{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "final_bert_long_docs.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ArmandDS/bert_for_long_text/blob/master/final_bert_long_docs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ojSZ5hvYnvV",
        "colab_type": "text"
      },
      "source": [
        "# Importing Necessary Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SlW7YHjGz0o5",
        "colab_type": "code",
        "outputId": "1120051f-c792-4568-ac1f-f469bf4e4044",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "np.random.seed(1337)\n",
        "from keras import Sequential\n",
        "from keras.utils import Sequence\n",
        "from keras.layers import LSTM, Dense, Masking\n",
        "import numpy as np\n",
        "import keras\n",
        "from keras.utils import np_utils\n",
        "from keras import optimizers\n",
        "from keras.models import Sequential, Model\n",
        "from keras.layers import Embedding, Dense, Input, concatenate, Layer, Lambda, Dropout, Activation\n",
        "import datetime\n",
        "from datetime import datetime\n",
        "from keras.callbacks import ModelCheckpoint, EarlyStopping, Callback, TensorBoard\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3JEJQ70-Ze4X",
        "colab_type": "text"
      },
      "source": [
        "# Loading The Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "82TK3tDYzpqm",
        "colab_type": "code",
        "outputId": "af1cc890-7712-44ed-be0d-b30a10b4b081",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 428
        }
      },
      "source": [
        "train_raw = pd.read_csv('consumer_complaints.csv')\n",
        "train_raw.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (5,11) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  interactivity=interactivity, compiler=compiler, result=result)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>date_received</th>\n",
              "      <th>product</th>\n",
              "      <th>sub_product</th>\n",
              "      <th>issue</th>\n",
              "      <th>sub_issue</th>\n",
              "      <th>consumer_complaint_narrative</th>\n",
              "      <th>company_public_response</th>\n",
              "      <th>company</th>\n",
              "      <th>state</th>\n",
              "      <th>zipcode</th>\n",
              "      <th>tags</th>\n",
              "      <th>consumer_consent_provided</th>\n",
              "      <th>submitted_via</th>\n",
              "      <th>date_sent_to_company</th>\n",
              "      <th>company_response_to_consumer</th>\n",
              "      <th>timely_response</th>\n",
              "      <th>consumer_disputed?</th>\n",
              "      <th>complaint_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>08/30/2013</td>\n",
              "      <td>Mortgage</td>\n",
              "      <td>Other mortgage</td>\n",
              "      <td>Loan modification,collection,foreclosure</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>U.S. Bancorp</td>\n",
              "      <td>CA</td>\n",
              "      <td>95993</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Referral</td>\n",
              "      <td>09/03/2013</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>511074</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>08/30/2013</td>\n",
              "      <td>Mortgage</td>\n",
              "      <td>Other mortgage</td>\n",
              "      <td>Loan servicing, payments, escrow account</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Wells Fargo &amp; Company</td>\n",
              "      <td>CA</td>\n",
              "      <td>91104</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Referral</td>\n",
              "      <td>09/03/2013</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>511080</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>08/30/2013</td>\n",
              "      <td>Credit reporting</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Incorrect information on credit report</td>\n",
              "      <td>Account status</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Wells Fargo &amp; Company</td>\n",
              "      <td>NY</td>\n",
              "      <td>11764</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Postal mail</td>\n",
              "      <td>09/18/2013</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>510473</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>08/30/2013</td>\n",
              "      <td>Student loan</td>\n",
              "      <td>Non-federal student loan</td>\n",
              "      <td>Repaying your loan</td>\n",
              "      <td>Repaying your loan</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Navient Solutions, Inc.</td>\n",
              "      <td>MD</td>\n",
              "      <td>21402</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Email</td>\n",
              "      <td>08/30/2013</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>510326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>08/30/2013</td>\n",
              "      <td>Debt collection</td>\n",
              "      <td>Credit card</td>\n",
              "      <td>False statements or representation</td>\n",
              "      <td>Attempted to collect wrong amount</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Resurgent Capital Services L.P.</td>\n",
              "      <td>GA</td>\n",
              "      <td>30106</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Web</td>\n",
              "      <td>08/30/2013</td>\n",
              "      <td>Closed with explanation</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>511067</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  date_received           product  ... consumer_disputed? complaint_id\n",
              "0    08/30/2013          Mortgage  ...                Yes       511074\n",
              "1    08/30/2013          Mortgage  ...                Yes       511080\n",
              "2    08/30/2013  Credit reporting  ...                 No       510473\n",
              "3    08/30/2013      Student loan  ...                Yes       510326\n",
              "4    08/30/2013   Debt collection  ...                Yes       511067\n",
              "\n",
              "[5 rows x 18 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S6kyJH661det",
        "colab_type": "code",
        "outputId": "6ab0745a-f50b-43fa-b0a8-ad44a9e6dffc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_raw.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(555957, 18)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Hr1v_VoZqws",
        "colab_type": "text"
      },
      "source": [
        "# Preprocessing Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wbxSBv8m6XaR",
        "colab_type": "text"
      },
      "source": [
        "Select non null:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wmb8-t51zx0n",
        "colab_type": "code",
        "outputId": "5156b4fd-d78f-43e2-ca3a-37c0bc20e8b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_raw = train_raw[train_raw.consumer_complaint_narrative.notnull()]\n",
        "train_raw.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(66806, 18)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pAMdnQBv1qlC",
        "colab_type": "code",
        "outputId": "b0cad918-406f-484e-a922-695ce9c01ab3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "train_raw.consumer_complaint_narrative.apply(lambda x: len(x.split())).plot(kind='hist')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f1fd238b240>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAD4CAYAAAAtrdtxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAW00lEQVR4nO3df/BddX3n8efLIIg/EyRlswltombs\nxu4KmGIcu7sWKwRsBTvUhXFKSlnTWWFWt86soe4s/mIHdltp2VEUS7Zg1YCIkkXcbKRMO/2DH0Ep\nEJDmK2BJ5MdXQKjaBcH3/nE/33j9+k1yc5L7vd9rno+ZO99z3uece97nJPm+cn7cc1NVSJLUxfNG\n3YAkaXwZIpKkzgwRSVJnhogkqTNDRJLU2UGjbmC2HX744bV06dJRtyFJY+W22277blUtnF4/4EJk\n6dKlbNmyZdRtSNJYSfLtmeqezpIkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLUmSEiSerM\nEJEkdXbAfWJ9Xyxd95WRrPeBC946kvVK0p54JCJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeG\niCSpM0NEktSZISJJ6mxoIZLkBUluSfJ3SbYm+VCrL0tyc5KJJFcmObjVD2njE2360r73OrfV701y\nQl99datNJFk3rG2RJM1smEciTwPHVdVrgaOA1UlWARcCF1XVq4AngLPa/GcBT7T6RW0+kqwATgNe\nA6wGPpFkXpJ5wMeBE4EVwOltXknSLBlaiFTP99vo89urgOOAq1v9cuCUNnxyG6dNf3OStPqGqnq6\nqu4HJoBj22uiqu6rqmeADW1eSdIsGeo1kXbEcDvwKLAZ+Bbwvap6ts2yHVjchhcDDwK06U8CL++v\nT1tmV/WZ+libZEuSLZOTk/tj0yRJDDlEquq5qjoKWELvyOGXh7m+3fRxaVWtrKqVCxcuHEULkvRz\naVbuzqqq7wE3Am8A5ieZegT9EmBHG94BHAnQpr8MeKy/Pm2ZXdUlSbNkmHdnLUwyvw0fCrwFuIde\nmJzaZlsDXNuGN7Zx2vS/qqpq9dPa3VvLgOXALcCtwPJ2t9fB9C6+bxzW9kiSftYwv5RqEXB5u4vq\necBVVXVdkruBDUk+CnwDuKzNfxnwmSQTwOP0QoGq2prkKuBu4Fng7Kp6DiDJOcAmYB6wvqq2DnF7\nJEnTDC1EquoO4OgZ6vfRuz4yvf7/gN/ZxXudD5w/Q/164Pp9blaS1ImfWJckdWaISJI6M0QkSZ0Z\nIpKkzgwRSVJnhogkqTNDRJLUmSEiSerMEJEkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLU\nmSEiSerMEJEkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLU2dBCJMmRSW5McneSrUne0+of\nTLIjye3tdVLfMucmmUhyb5IT+uqrW20iybq++rIkN7f6lUkOHtb2SJJ+1jCPRJ4F3ldVK4BVwNlJ\nVrRpF1XVUe11PUCbdhrwGmA18Ikk85LMAz4OnAisAE7ve58L23u9CngCOGuI2yNJmmZoIVJVD1XV\n19vwPwL3AIt3s8jJwIaqerqq7gcmgGPba6Kq7quqZ4ANwMlJAhwHXN2Wvxw4ZThbI0mayaxcE0my\nFDgauLmVzklyR5L1SRa02mLgwb7FtrfaruovB75XVc9Oq8+0/rVJtiTZMjk5uR+2SJIEsxAiSV4M\nfBF4b1U9BVwCvBI4CngI+JNh91BVl1bVyqpauXDhwmGvTpIOGAcN882TPJ9egHy2qq4BqKpH+qZ/\nGriuje4AjuxbfEmrsYv6Y8D8JAe1o5H++SVJs2CYd2cFuAy4p6o+1ldf1Dfb24G72vBG4LQkhyRZ\nBiwHbgFuBZa3O7EOpnfxfWNVFXAjcGpbfg1w7bC2R5L0s4Z5JPJG4HeBO5Pc3mp/RO/uqqOAAh4A\n/gCgqrYmuQq4m96dXWdX1XMASc4BNgHzgPVVtbW93/uBDUk+CnyDXmhJkmbJ0EKkqv4WyAyTrt/N\nMucD589Qv36m5arqPnp3b0mSRsBPrEuSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCR\nJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOjNEJEmdGSKSpM4M\nEUlSZ4aIJKkzQ0SS1JkhIknqbKAQSfIv9/aNkxyZ5MYkdyfZmuQ9rX5Yks1JtrWfC1o9SS5OMpHk\njiTH9L3Xmjb/tiRr+uqvS3JnW+biJNnbPiVJ3Q16JPKJJLckeXeSlw24zLPA+6pqBbAKODvJCmAd\ncENVLQduaOMAJwLL22stcAn0Qgc4D3g9cCxw3lTwtHne1bfc6gF7kyTtBwOFSFX9a+CdwJHAbUk+\nl+Qte1jmoar6ehv+R+AeYDFwMnB5m+1y4JQ2fDJwRfXcBMxPsgg4AdhcVY9X1RPAZmB1m/bSqrqp\nqgq4ou+9JEmzYOBrIlW1DfgvwPuBfwtcnOSbSX57T8smWQocDdwMHFFVD7VJDwNHtOHFwIN9i21v\ntd3Vt89QlyTNkkGvifyrJBfRO5o4DvitqvoXbfiiPSz7YuCLwHur6qn+ae0Ioro0vjeSrE2yJcmW\nycnJYa9Okg4Ygx6J/E/g68Brq+rsvtNU36F3dDKjJM+nFyCfraprWvmRdiqK9vPRVt9B73TZlCWt\ntrv6khnqP6OqLq2qlVW1cuHChQNsriRpEIOGyFuBz1XVPwEkeV6SFwJU1WdmWqDdKXUZcE9Vfaxv\n0kZg6g6rNcC1ffUz2l1aq4An22mvTcDxSRa0C+rHA5vatKeSrGrrOqPvvSRJs2DQEPkacGjf+Atb\nbXfeCPwucFyS29vrJOAC4C1JtgG/0cYBrgfuAyaATwPvBqiqx4GPALe214dbjTbPn7dlvgV8dcDt\nkSTtBwcNON8Lqur7UyNV9f2pI5Fdqaq/BXb1uY03zzB/AWfv4r3WA+tnqG8BfmV3fUiShmfQI5Ef\nTPvw3+uAfxpOS5KkcTHokch7gS8k+Q69o4t/Bvy7oXUlSRoLA4VIVd2a5JeBV7fSvVX1o+G1JUka\nB4MeiQD8KrC0LXNMEqrqiqF0JUkaCwOFSJLPAK8Ebgeea+WpR41Ikg5Qgx6JrARWtDuoNMuWrvvK\nyNb9wAVvHdm6Jc19g96ddRe9i+mSJO006JHI4cDdSW4Bnp4qVtXbhtKVJGksDBoiHxxmE5Kk8TTo\nLb5/neSXgOVV9bX2afV5w21NkjTXDfoo+HcBVwOfaqXFwJeH1ZQkaTwMemH9bHoPVHwKdn5B1S8M\nqylJ0ngYNESerqpnpkaSHMQsfJmUJGluGzRE/jrJHwGHtu9W/wLwv4fXliRpHAwaIuuASeBO4A/o\nfffHLr/RUJJ0YBj07qwf0/uiqE8Ptx1J0jgZ9NlZ9zPDNZCqesV+70iSNDb25tlZU14A/A5w2P5v\nR5I0Tga6JlJVj/W9dlTVnwI+mU+SDnCDns46pm/0efSOTPbmu0gkST+HBg2CP+kbfhZ4AHjHfu9G\nkjRWBr0769eH3YgkafwMejrrD3c3vao+tn/akSSNk725O+tXgY1t/LeAW4Btw2hKkjQeBv3E+hLg\nmKp6X1W9D3gd8ItV9aGq+tBMCyRZn+TRJHf11T6YZEeS29vrpL5p5yaZSHJvkhP66qtbbSLJur76\nsiQ3t/qVSQ7e242XJO2bQUPkCOCZvvFnWm13/gJYPUP9oqo6qr2uB0iyAjgNeE1b5hNJ5iWZB3wc\nOBFYAZze5gW4sL3Xq4AngLMG3BZJ0n4yaIhcAdzSjiQ+CNwMXL67Barqb4DHB3z/k4ENVfV0Vd0P\nTADHttdEVd3XniK8ATg5SYDj6H3HCa2XUwZclyRpPxn0w4bnA2fS+x//E8CZVfXfOq7znCR3tNNd\nC1ptMfBg3zzbW21X9ZcD36uqZ6fVZ5RkbZItSbZMTk52bFuSNN2gRyIALwSeqqo/A7YnWdZhfZcA\nrwSOAh7ipz9/MjRVdWlVrayqlQsXLpyNVUrSAWHQr8c9D3g/cG4rPR/4y71dWVU9UlXP9T0V+Ng2\naQdwZN+sS1ptV/XHgPnty7H665KkWTTokcjbgbcBPwCoqu8AL9nblSVZNO09p+7c2gicluSQdoSz\nnN4txLcCy9udWAfTu/i+saoKuBE4tS2/Brh2b/uRJO2bQT8n8kxVVZICSPKiPS2Q5PPAm4DDk2wH\nzgPelOQoeo+Vf4DeF1xRVVuTXAXcTe+xKmdX1XPtfc4BNgHzgPVVtbWt4v3AhiQfBb4BXDbgtkiS\n9pNBQ+SqJJ+idwrpXcDvs4cvqKqq02co7/IXfbt4f/4M9evpfZPi9Pp9/OR0mCRpBAZ9dtYft+9W\nfwp4NfBfq2rzUDuTJM15ewyR9oG/r7WHMBockqSd9nhhvV2b+HGSl81CP5KkMTLoNZHvA3cm2Uy7\nQwugqv7jULqSJI2FQUPkmvaSJGmn3YZIkl+sqn+oqt0+J0uSdGDa0zWRL08NJPnikHuRJI2ZPYVI\n+oZfMcxGJEnjZ08hUrsYliRpjxfWX5vkKXpHJIe2Ydp4VdVLh9qdJGlO222IVNW82WpEkjR+9ub7\nRCRJ+imGiCSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ\n6mxoIZJkfZJHk9zVVzssyeYk29rPBa2eJBcnmUhyR5Jj+pZZ0+bflmRNX/11Se5sy1ycJEiSZtUw\nj0T+Alg9rbYOuKGqlgM3tHGAE4Hl7bUWuAR6oQOcB7weOBY4byp42jzv6ltu+rokSUM2tBCpqr8B\nHp9WPhmY+r72y4FT+upXVM9NwPwki4ATgM1V9XhVPQFsBla3aS+tqpuqqoAr+t5LkjRLZvuayBFV\n9VAbfhg4og0vBh7sm297q+2uvn2G+oySrE2yJcmWycnJfdsCSdJOI7uw3o4gZuUrd6vq0qpaWVUr\nFy5cOBurlKQDwmyHyCPtVBTt56OtvgM4sm++Ja22u/qSGeqSpFk02yGyEZi6w2oNcG1f/Yx2l9Yq\n4Ml22msTcHySBe2C+vHApjbtqSSr2l1ZZ/S9lyRpluz2O9b3RZLPA28CDk+ynd5dVhcAVyU5C/g2\n8I42+/XAScAE8EPgTICqejzJR4Bb23wfrqqpi/XvpncH2KHAV9tLkjSLhhYiVXX6Lia9eYZ5Czh7\nF++zHlg/Q30L8Cv70qMkad/4iXVJUmeGiCSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnQ3tcyL6\n+bB03VdGst4HLnjrSNYrae94JCJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZ\nISJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSps5GESJIHktyZ5PYkW1rtsCSbk2xrPxe0\nepJcnGQiyR1Jjul7nzVt/m1J1oxiWyTpQDbKI5Ffr6qjqmplG18H3FBVy4Eb2jjAicDy9loLXAK9\n0AHOA14PHAucNxU8kqTZMZdOZ50MXN6GLwdO6atfUT03AfOTLAJOADZX1eNV9QSwGVg9201L0oFs\nVCFSwP9NcluSta12RFU91IYfBo5ow4uBB/uW3d5qu6r/jCRrk2xJsmVycnJ/bYMkHfBG9R3rv1ZV\nO5L8ArA5yTf7J1ZVJan9tbKquhS4FGDlypX77X0l6UA3kiORqtrRfj4KfIneNY1H2mkq2s9H2+w7\ngCP7Fl/SaruqS5JmyayHSJIXJXnJ1DBwPHAXsBGYusNqDXBtG94InNHu0loFPNlOe20Cjk+yoF1Q\nP77VJEmzZBSns44AvpRkav2fq6r/k+RW4KokZwHfBt7R5r8eOAmYAH4InAlQVY8n+Qhwa5vvw1X1\n+OxthiRp1kOkqu4DXjtD/THgzTPUCzh7F++1Hli/v3uUJA1mLt3iK0kaM4aIJKmzUd3iK+3W0nVf\nGdm6H7jgrSNbtzRuPBKRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzA8bStOM\n6oOOfshR48gjEUlSZx6JSHOEj3rROPJIRJLUmSEiSerMEJEkdWaISJI6M0QkSZ0ZIpKkzgwRSVJn\nhogkqTNDRJLUmSEiSeps7EMkyeok9yaZSLJu1P1I0oFkrEMkyTzg48CJwArg9CQrRtuVJB04xv0B\njMcCE1V1H0CSDcDJwN0j7UoaMz7+Xl2Ne4gsBh7sG98OvH76TEnWAmvb6PeT3NtxfYcD3+247KiN\nc+8w3v3b+y7kwmG9807u+/3nl2YqjnuIDKSqLgUu3df3SbKlqlbuh5Zm3Tj3DuPdv72Pzjj3Py69\nj/U1EWAHcGTf+JJWkyTNgnEPkVuB5UmWJTkYOA3YOOKeJOmAMdans6rq2STnAJuAecD6qto6xFXu\n8ymxERrn3mG8+7f30Rnn/sei91TVqHuQJI2pcT+dJUkaIUNEktSZITKAuf5olSRHJrkxyd1JtiZ5\nT6sflmRzkm3t54JWT5KL2/bckeSY0W5BT5J5Sb6R5Lo2vizJza3PK9vNEyQ5pI1PtOlLR9z3/CRX\nJ/lmknuSvGGc9n2S/9T+3tyV5PNJXjBX932S9UkeTXJXX22v93WSNW3+bUnWjLj//9H+7tyR5EtJ\n5vdNO7f1f2+SE/rqc+d3UlX52s2L3gX7bwGvAA4G/g5YMeq+pvW4CDimDb8E+Ht6j4H578C6Vl8H\nXNiGTwK+CgRYBdw86m1off0h8DngujZ+FXBaG/4k8B/a8LuBT7bh04ArR9z35cC/b8MHA/PHZd/T\n+8Du/cChffv89+bqvgf+DXAMcFdfba/2NXAYcF/7uaANLxhh/8cDB7XhC/v6X9F+3xwCLGu/h+bN\ntd9JI/vLOy4v4A3Apr7xc4FzR93XHnq+FngLcC+wqNUWAfe24U8Bp/fNv3O+Efa8BLgBOA64rv3D\n/27fP66dfw707sZ7Qxs+qM2XEfX9svZLONPqY7Hv+clTHw5r+/I64IS5vO+BpdN+Ce/VvgZOBz7V\nV/+p+Wa7/2nT3g58tg3/1O+aqX0/134neTprz2Z6tMriEfWyR+30wtHAzcARVfVQm/QwcEQbnovb\n9KfAfwZ+3MZfDnyvqp5t4/097uy/TX+yzT8Ky4BJ4H+1U3F/nuRFjMm+r6odwB8D/wA8RG9f3sZ4\n7Pspe7uv59SfwTS/T+/oCcakf0Pk50iSFwNfBN5bVU/1T6vef1nm5P3cSX4TeLSqbht1Lx0cRO/0\nxCVVdTTwA3qnVHaa4/t+Ab2Hli4D/jnwImD1SJvaB3N5X+9Jkg8AzwKfHXUve8MQ2bOxeLRKkufT\nC5DPVtU1rfxIkkVt+iLg0Vafa9v0RuBtSR4ANtA7pfVnwPwkUx+I7e9xZ/9t+suAx2az4T7bge1V\ndXMbv5peqIzLvv8N4P6qmqyqHwHX0PvzGId9P2Vv9/Vc+zMgye8Bvwm8swUhjEn/hsiezflHqyQJ\ncBlwT1V9rG/SRmDqzpM19K6VTNXPaHevrAKe7DsdMOuq6tyqWlJVS+nt37+qqncCNwKnttmm9z+1\nXae2+Ufyv8+qehh4MMmrW+nN9L6KYCz2Pb3TWKuSvLD9PZrqf87v+z57u683AccnWdCOxI5vtZFI\nspreqdy3VdUP+yZtBE5rd8QtA5YDtzDXfieN6mLMOL3o3eXx9/TuiPjAqPuZob9fo3cIfwdwe3ud\nRO9c9Q3ANuBrwGFt/tD7Mq9vAXcCK0e9DX3b8iZ+cnfWK+j9o5kAvgAc0uovaOMTbforRtzzUcCW\ntv+/TO+On7HZ98CHgG8CdwGfoXc30Jzc98Dn6V27+RG9o8CzuuxretceJtrrzBH3P0HvGsfUv91P\n9s3/gdb/vcCJffU58zvJx55IkjrzdJYkqTNDRJLUmSEiSerMEJEkdWaISJI6M0QkSZ0ZIpKkzv4/\nfaXl6ul9K0gAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eGyH4kwB0TH6",
        "colab_type": "code",
        "outputId": "c8117d31-a15b-49f4-d6e0-8e9b905a2c65",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        }
      },
      "source": [
        "train_raw['len_txt'] =train_raw.consumer_complaint_narrative.apply(lambda x: len(x.split()))\n",
        "train_raw.describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>complaint_id</th>\n",
              "      <th>len_txt</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>6.680600e+04</td>\n",
              "      <td>66806.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.571665e+06</td>\n",
              "      <td>190.644014</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1.545692e+05</td>\n",
              "      <td>166.830597</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.290181e+06</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>1.443264e+06</td>\n",
              "      <td>71.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>1.569485e+06</td>\n",
              "      <td>136.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>1.702750e+06</td>\n",
              "      <td>254.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.888608e+06</td>\n",
              "      <td>1284.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       complaint_id       len_txt\n",
              "count  6.680600e+04  66806.000000\n",
              "mean   1.571665e+06    190.644014\n",
              "std    1.545692e+05    166.830597\n",
              "min    1.290181e+06      1.000000\n",
              "25%    1.443264e+06     71.000000\n",
              "50%    1.569485e+06    136.000000\n",
              "75%    1.702750e+06    254.000000\n",
              "max    1.888608e+06   1284.000000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LqFgGC_TkvyA",
        "colab_type": "code",
        "outputId": "db5dff6d-862a-45cf-a71c-957c81cb5888",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_raw.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(66806, 19)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SumN633drVVO",
        "colab_type": "text"
      },
      "source": [
        "Select only the row with number of words greater than 250:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "maZQzkqv0iFa",
        "colab_type": "code",
        "outputId": "ceb3600d-cc69-478a-d5a9-b45ce797bd61",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_raw = train_raw[train_raw.len_txt >249]\n",
        "train_raw.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(17142, 19)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WzYJ1SYn0uE6",
        "colab_type": "code",
        "outputId": "96884ed2-2088-46d1-d026-26000441690d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train_raw = train_raw[['consumer_complaint_narrative', 'product']]\n",
        "train_raw.reset_index(inplace=True, drop=True)\n",
        "train_raw.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>consumer_complaint_narrative</th>\n",
              "      <th>product</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>In XX/XX/XXXX my wages that I earned at my job...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>XXXX was submitted XX/XX/XXXX. At the time I s...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I spoke to XXXX of green tree representatives ...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>i opened XXXX Bank of America credit cards 15-...</td>\n",
              "      <td>Credit card</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I applied for a loan with XXXX XXXX and had pu...</td>\n",
              "      <td>Consumer Loan</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        consumer_complaint_narrative        product\n",
              "0  In XX/XX/XXXX my wages that I earned at my job...       Mortgage\n",
              "1  XXXX was submitted XX/XX/XXXX. At the time I s...       Mortgage\n",
              "2  I spoke to XXXX of green tree representatives ...       Mortgage\n",
              "3  i opened XXXX Bank of America credit cards 15-...    Credit card\n",
              "4  I applied for a loan with XXXX XXXX and had pu...  Consumer Loan"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "beKGErd96q9p",
        "colab_type": "text"
      },
      "source": [
        "Group similar products"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6YzGjz1o01x2",
        "colab_type": "code",
        "outputId": "317ccd3e-52b4-4cce-da32-3841548d5e7b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train_raw.at[train_raw['product'] == 'Credit reporting', 'product'] = 'Credit reporting, credit repair services, or other personal consumer reports'\n",
        "train_raw.at[train_raw['product'] == 'Credit card', 'product'] = 'Credit card or prepaid card'\n",
        "train_raw.at[train_raw['product'] == 'Prepaid card', 'product'] = 'Credit card or prepaid card'\n",
        "train_raw.at[train_raw['product'] == 'Payday loan', 'product'] = 'Payday loan, title loan, or personal loan'\n",
        "train_raw.at[train_raw['product'] == 'Virtual currency', 'product'] = 'Money transfer, virtual currency, or money service'\n",
        "train_raw.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>consumer_complaint_narrative</th>\n",
              "      <th>product</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>In XX/XX/XXXX my wages that I earned at my job...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>XXXX was submitted XX/XX/XXXX. At the time I s...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I spoke to XXXX of green tree representatives ...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>i opened XXXX Bank of America credit cards 15-...</td>\n",
              "      <td>Credit card or prepaid card</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I applied for a loan with XXXX XXXX and had pu...</td>\n",
              "      <td>Consumer Loan</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                        consumer_complaint_narrative                      product\n",
              "0  In XX/XX/XXXX my wages that I earned at my job...                     Mortgage\n",
              "1  XXXX was submitted XX/XX/XXXX. At the time I s...                     Mortgage\n",
              "2  I spoke to XXXX of green tree representatives ...                     Mortgage\n",
              "3  i opened XXXX Bank of America credit cards 15-...  Credit card or prepaid card\n",
              "4  I applied for a loan with XXXX XXXX and had pu...                Consumer Loan"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4jG9xKs1CJnh",
        "colab_type": "code",
        "outputId": "57102e86-62ab-4dc0-d4fd-f48020ecf525",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "for l in np.unique(train_raw['product']):\n",
        "  print(l)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bank account or service\n",
            "Consumer Loan\n",
            "Credit card or prepaid card\n",
            "Credit reporting, credit repair services, or other personal consumer reports\n",
            "Debt collection\n",
            "Money transfers\n",
            "Mortgage\n",
            "Other financial service\n",
            "Payday loan, title loan, or personal loan\n",
            "Student loan\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ISkKV5RX344m",
        "colab_type": "code",
        "outputId": "225b6441-e028-4983-c10d-cd2a3f45cbc6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 648
        }
      },
      "source": [
        "train_raw['product'].value_counts().sort_values(ascending=False).plot(kind='bar')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f1fd1e881d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAJmCAYAAACwgFCUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd5xdVbn/8c83Cb0GCREpBgVBQJoB\ngiBSpIk0aSIlIIp6UVHv9Sd4r6Kg0myAVy5IMXCRDhIBwRiqIIQAocMlNAEpkRKQTvj+/ljrJCfD\nlCCz1z6c/bxfr3ll9jpnsp9JZp6zzirPkm1CCCE0w5C6AwghhFBOJP0QQmiQSPohhNAgkfRDCKFB\nIumHEEKDRNIPIYQGGVZ3AP1ZfPHFPWrUqLrDCCGEd5WbbrrpH7ZH9PZYRyf9UaNGMXny5LrDCCGE\ndxVJD/f1WAzvhBBCg0TSDyGEBomkH0IIDRJJP4QQGiSSfgghNEgk/RBCaJA5SvqSFpV0rqR7JN0t\naT1Ji0maIOm+/Ofw/FxJOkbSVEm3SVqr7e8Zm59/n6SxVX1TIYQQejenPf2jgUttrwSsDtwNHAhM\ntL0CMDFfA2wFrJA/9gOOA5C0GHAwsC6wDnBw64UihBBCGQNuzpK0CLAhsDeA7deA1yRtB2yUnzYO\nuBL4DrAdcKrT6SzX53cJS+bnTrD9TP57JwBbAme8k29g1IEXv5Mv56HDt35HXx9CCO8mc9LTXw6Y\nBpwi6RZJJ0paABhp+/H8nCeAkfnzpYBH2r7+0dzWV3sIIYRC5iTpDwPWAo6zvSbwIrOGcgDIvfpB\nOXdR0n6SJkuaPG3atMH4K0MIIWRzkvQfBR61fUO+Ppf0IvBkHrYh//lUfvwxYJm2r186t/XVPhvb\nJ9gebXv0iBG91gsKIYTwLxow6dt+AnhE0oq5aVPgLmA80FqBMxa4MH8+Htgrr+IZA0zPw0CXAZtL\nGp4ncDfPbSGEEAqZ0yqbXwNOlzQ38ACwD+kF42xJ+wIPA7vk514CfAqYCryUn4vtZyQdCtyYn3dI\na1I3hBBCGXOU9G1PAUb38tCmvTzXwP59/D0nAye/nQBDCCEMntiRG0IIDRJJP4QQGiSSfgghNEgk\n/RBCaJBI+iGE0CCR9EMIoUEi6YcQQoNE0g8hhAaJpB9CCA0SST+EEBokkn4IITRIJP0QQmiQSPoh\nhNAgkfRDCKFBIumHEEKDRNIPIYQGiaQfQggNEkk/hBAaJJJ+CCE0SCT9EEJokEj6IYTQIJH0Qwih\nQSLphxBCg0TSDyGEBomkH0IIDTJHSV/SQ5JulzRF0uTctpikCZLuy38Oz+2SdIykqZJuk7RW298z\nNj//Pkljq/mWQggh9OXt9PQ3tr2G7dH5+kBgou0VgIn5GmArYIX8sR9wHKQXCeBgYF1gHeDg1gtF\nCCGEMt7J8M52wLj8+Thg+7b2U51cDywqaUlgC2CC7WdsPwtMALZ8B/cPIYTwNs1p0jfwJ0k3Sdov\nt420/Xj+/AlgZP58KeCRtq99NLf11R5CCKGQYXP4vA1sPyZpCWCCpHvaH7RtSR6MgPKLyn4Ayy67\n7GD8lSGEELI56unbfiz/+RRwAWlM/sk8bEP+86n89MeAZdq+fOnc1ld7z3udYHu07dEjRox4e99N\nCCGEfg2Y9CUtIGmh1ufA5sAdwHigtQJnLHBh/nw8sFdexTMGmJ6HgS4DNpc0PE/gbp7bQgghFDIn\nwzsjgQsktZ7/O9uXSroROFvSvsDDwC75+ZcAnwKmAi8B+wDYfkbSocCN+XmH2H5m0L6TEEIIAxow\n6dt+AFi9l/angU17aTewfx9/18nAyW8/zBBCCIMhduSGEEKDRNIPIYQGiaQfQggNEkk/hBAaJJJ+\nCCE0SCT9EEJokEj6IYTQIJH0QwihQSLphxBCg0TSDyGEBomkH0IIDRJJP4QQGiSSfgghNEgk/RBC\naJBI+iGE0CCR9EMIoUEi6YcQQoNE0g8hhAaJpB9CCA0SST+EEBokkn4IITRIJP0QQmiQSPohhNAg\nkfRDCKFBIumHEEKDRNIPIYQGmeOkL2mopFskXZSvl5N0g6Spks6SNHdunydfT82Pj2r7Ow7K7fdK\n2mKwv5kQQgj9ezs9/QOAu9uujwB+YXt54Flg39y+L/Bsbv9Ffh6SVgY+C6wCbAn8WtLQdxZ+CCGE\nt2OOkr6kpYGtgRPztYBNgHPzU8YB2+fPt8vX5Mc3zc/fDjjT9qu2HwSmAusMxjcRQghhzsxpT/+X\nwP8D3szX7wGes/1Gvn4UWCp/vhTwCEB+fHp+/sz2Xr5mJkn7SZosafK0adPexrcSQghhIAMmfUmf\nBp6yfVOBeLB9gu3RtkePGDGixC1DCKExhs3Bc9YHtpX0KWBeYGHgaGBRScNyb35p4LH8/MeAZYBH\nJQ0DFgGebmtvaf+aEEIIBQzY07d9kO2lbY8iTcRebnt34Apgp/y0scCF+fPx+Zr8+OW2nds/m1f3\nLAesAEwatO8khBDCgOakp9+X7wBnSvoRcAtwUm4/CThN0lTgGdILBbbvlHQ2cBfwBrC/7Rnv4P4h\nhBDepreV9G1fCVyZP3+AXlbf2H4F2LmPr/8x8OO3G2QIIYTBETtyQwihQSLphxBCg0TSDyGEBomk\nH0IIDRJJP4QQGiSSfgghNEgk/RBCaJBI+iGE0CCR9EMIoUEi6YcQQoNE0g8hhAaJpB9CCA0SST+E\nEBokkn4IITRIJP0QQmiQSPohhNAgkfRDCKFBIumHEEKDRNIPIYQGiaQfQggNEkk/hBAaJJJ+CCE0\nSCT9EEJokEj6IYTQIJH0QwihQQZM+pLmlTRJ0q2S7pT0w9y+nKQbJE2VdJakuXP7PPl6an58VNvf\ndVBuv1fSFlV9UyGEEHo3Jz39V4FNbK8OrAFsKWkMcATwC9vLA88C++bn7ws8m9t/kZ+HpJWBzwKr\nAFsCv5Y0dDC/mRBCCP0bMOk7+We+nCt/GNgEODe3jwO2z59vl6/Jj28qSbn9TNuv2n4QmAqsMyjf\nRQghhDkyR2P6koZKmgI8BUwA7gees/1GfsqjwFL586WARwDy49OB97S39/I1IYQQCpijpG97hu01\ngKVJvfOVqgpI0n6SJkuaPG3atKpuE0IIjfS2Vu/Yfg64AlgPWFTSsPzQ0sBj+fPHgGUA8uOLAE+3\nt/fyNe33OMH2aNujR4wY8XbCCyGEMIA5Wb0zQtKi+fP5gM2Au0nJf6f8tLHAhfnz8fma/Pjltp3b\nP5tX9ywHrABMGqxvJIQQwsCGDfwUlgTG5ZU2Q4CzbV8k6S7gTEk/Am4BTsrPPwk4TdJU4BnSih1s\n3ynpbOAu4A1gf9szBvfbCSGE0J8Bk77t24A1e2l/gF5W39h+Bdi5j7/rx8CP336YIYQQBsOc9PTD\nAEYdePE7/jseOnzrQYgkhBD6F2UYQgihQSLphxBCg0TSDyGEBomkH0IIDRJJP4QQGiSSfgghNEgk\n/RBCaJBI+iGE0CCR9EMIoUEi6YcQQoNE0g8hhAaJpB9CCA0SBde6yDst/BZF30LoftHTDyGEBomk\nH0IIDRJJP4QQGiSSfgghNEgk/RBCaJBI+iGE0CCR9EMIoUEi6YcQQoNE0g8hhAaJpB9CCA0SST+E\nEBpkwKQvaRlJV0i6S9Kdkg7I7YtJmiDpvvzn8NwuScdImirpNklrtf1dY/Pz75M0trpvK4QQQm/m\npKf/BvDvtlcGxgD7S1oZOBCYaHsFYGK+BtgKWCF/7AccB+lFAjgYWBdYBzi49UIRQgihjAGTvu3H\nbd+cP38BuBtYCtgOGJefNg7YPn++HXCqk+uBRSUtCWwBTLD9jO1ngQnAloP63YQQQujX2xrTlzQK\nWBO4ARhp+/H80BPAyPz5UsAjbV/2aG7rqz2EEEIhc5z0JS0InAd8w/bz7Y/ZNuDBCEjSfpImS5o8\nbdq0wfgrQwghZHOU9CXNRUr4p9s+Pzc/mYdtyH8+ldsfA5Zp+/Klc1tf7bOxfYLt0bZHjxgx4u18\nLyGEEAYw4MlZkgScBNxt++dtD40HxgKH5z8vbGv/qqQzSZO2020/Luky4Cdtk7ebAwcNzrcROsU7\nPb0L4gSvEKo0J8clrg/sCdwuaUpu+y4p2Z8taV/gYWCX/NglwKeAqcBLwD4Atp+RdChwY37eIbaf\nGZTvIoQQwhwZMOnb/gugPh7etJfnG9i/j7/rZODktxNgCCGEwRM7ckMIoUEi6YcQQoNE0g8hhAaZ\nk4ncEN513ukqolhBFLpV9PRDCKFBIumHEEKDRNIPIYQGiaQfQggNEkk/hBAaJJJ+CCE0SCT9EEJo\nkEj6IYTQIJH0QwihQSLphxBCg0TSDyGEBomkH0IIDRJJP4QQGiSSfgghNEgk/RBCaJBI+iGE0CCR\n9EMIoUEi6YcQQoNE0g8hhAaJpB9CCA0SST+EEBpkwKQv6WRJT0m6o61tMUkTJN2X/xye2yXpGElT\nJd0maa22rxmbn3+fpLHVfDshhBD6Myc9/d8CW/ZoOxCYaHsFYGK+BtgKWCF/7AccB+lFAjgYWBdY\nBzi49UIRQgihnGEDPcH21ZJG9WjeDtgofz4OuBL4Tm4/1baB6yUtKmnJ/NwJtp8BkDSB9EJyxjv+\nDkLoUKMOvPgd/x0PHb71IEQSwiz/6pj+SNuP58+fAEbmz5cCHml73qO5ra/2EEIIBb3jidzcq/cg\nxAKApP0kTZY0edq0aYP114YQQuBfT/pP5mEb8p9P5fbHgGXanrd0buur/S1sn2B7tO3RI0aM+BfD\nCyGE0Jt/NemPB1orcMYCF7a175VX8YwBpudhoMuAzSUNzxO4m+e2EEIIBQ04kSvpDNJE7OKSHiWt\nwjkcOFvSvsDDwC756ZcAnwKmAi8B+wDYfkbSocCN+XmHtCZ1QwghlDMnq3d26+OhTXt5roH9+/h7\nTgZOflvRhRBCGFSxIzeEEBokkn4IITRIJP0QQmiQSPohhNAgA07khhDe3d5pOYgoBdFdoqcfQggN\nEkk/hBAaJJJ+CCE0SCT9EEJokEj6IYTQIJH0QwihQSLphxBCg0TSDyGEBomkH0IIDRJJP4QQGiSS\nfgghNEgk/RBCaJBI+iGE0CCR9EMIoUEi6YcQQoNE0g8hhAaJpB9CCA0SST+EEBokjksMIVTunR7Z\nCHFs42CJnn4IITRIJP0QQmiQ4sM7krYEjgaGAifaPrx0DCGEZnqnw0zdMMRUtKcvaSjw38BWwMrA\nbpJWLhlDCCE0Weme/jrAVNsPAEg6E9gOuKtwHCGEUIu6J7VLj+kvBTzSdv1obgshhFCAbJe7mbQT\nsKXtL+TrPYF1bX+17Tn7AfvlyxWBe9/hbRcH/vEO/47B0AlxdEIM0BlxRAyzdEIcnRADdEYcgxHD\n+22P6O2B0sM7jwHLtF0vndtmsn0CcMJg3VDSZNujB+vvezfH0QkxdEocEUNnxdEJMXRKHFXHUHp4\n50ZgBUnLSZob+CwwvnAMIYTQWEV7+rbfkPRV4DLSks2Tbd9ZMoYQQmiy4uv0bV8CXFLwloM2VPQO\ndUIcnRADdEYcEcMsnRBHJ8QAnRFHpTEUncgNIYRQryjDEEIIDRJJP4QQGqSrSytLmt/2SzXefyng\n/bT9O9u+usB9P9Pf47bPrzqGdpLmB/4dWNb2FyWtAKxo+6KCMewMXGr7BUn/BawF/Mj2zaVi6BSS\n1gd+wKyfTQG2/YGCMXwQeNT2q5I2AlYDTrX9XKkYchzzADsCo5j99/SQgjEI2B34gO1DJC0LvNf2\npEru141j+pI+BpwILGh7WUmrA1+y/W8FYzgC2JVUYmJGbrbtbQvc+5T86RLAx4DL8/XGwHW2P111\nDD3iOQu4CdjL9qr5ReA622sUjOE226tJ2gD4EXAU8H3b6xaM4TPAEaT/FzEr2S5cKoYcxz3AN0n/\nJ62fTWw/XTCGKcBoUrK9BLgQWMX2p0rFkOO4FJjOW/8tflYwhuOAN4FNbH9Y0nDgT7bXruJ+3drT\n/wWwBXkPgO1bJW1YOIbtSb3ZVwvfF9v7AEj6E7Cy7cfz9ZLAb0vHA3zQ9q6SdsvxvZR7NyW1fqG3\nBk6wfbGkHxWO4UhgG9t3F75vT9Nt/7HmGN7MS7h3AI61faykW2qIY2nbW9Zw33br2l6r9f3bfjbv\nY6pE147p236kR9OMXp9YnQeAuQrfs6dlWgk/exJYtoY4XpM0H2CY+da+9IvhY5KOJ737uiS/rS/9\n8/9kByR8gCskHSVpPUlrtT4Kx/B67gSMBVrDfHX8vlwn6SM13Lfd67kCcev3YwSp51+Jbu3pP5KH\neCxpLuAAoPQv20vAFEkTaUtwtr9eMIaJki4DzsjXuwJ/Lnj/loOBS4FlJJ0OrA/sXTiGXYAtgZ/a\nfi6/6/l24Rgm56Gu3zP7z0TRORagNaTVvtXfwCYFY9gH+DLwY9sPSloOOK3g/Vs2APaW9CDp/6Q1\n5LZawRiOAS4AlpD0Y2An4L+qulm3jukvTjqo5ZOk/8Q/AQcUHrMc21u77XGlYshx7AC0hrautn1B\nyfu3xfEeYAzp/+N620WLWkk6zfaeA7VVHMMpvTTb9udLxdAJcq/2VNu7d0As7++t3fbDheNYCdiU\n9Psxscp3hF2Z9DtFHpf7UL681/brBe89FPiz7Y1L3bOfWHYALrc9PV8vCmxk+/cFY7jZ9lpt10OB\n22038hAfSVsDqwDzttoKr1j5C2ni8rVS9+yPpCWY/d/ibwXvPQa40/YL+Xph4MO2b6jifl05vCPp\nmF6apwOTbV9YKIaNgHHAQ6RX72UkjS2xZBPA9gxJb0papJVsa3Rw+zuMPLxyMGmYo1KSDgK+C8wn\n6flWM/AahbfcS5oX2Je3JtuiPX1J/wPMT1rNdSJpOKGS5YH9eAC4VtJ44MVWo+2flwxC0rbAz4D3\nAU+RlrHeTfo/KuU40hLiln/20jZounUid15gDeC+/LEaqYzzvpJ+WSiGnwGb2/6E7Q1Jq4l+Ueje\nLf8Ebpd0kqRjWh+FY4Def86KdDhsHwYsQhpOWDh/LGT7PbYPKhFDm9OA95J+Fq4i/Uy+UDgGgI/Z\n3gt41vYPgfWY9Y60lPtJE7hDgIXaPko7lDTs+H+2lyMNsVxfOAa5bcjF9ptU+PvRlT19UpJf3/YM\nmLkO9hrSpM3thWKYy/bMA2Bs/1+eVC7p/PxRt8mSfk46Hxlgf9K66CJsvympkjXPb9PytneWtJ3t\ncZJ+R/q5LO3l/OdLkt4HPA0sWTKA/GJT+wZK4HXbT0saImmI7SsKdgxbHpD0dVLvHuDfSO+EKtGt\nSX84sCBpSAdgAWCxPORRaqngZEknAv+br3cHJhe6N1B+0rgfXwO+B5yVryeQEn9JN0ta2/aNhe/b\nrjWn85ykVYEnSBu1Srsoz6scBdxMWrnzm5IBSFoPOIn0e1rLBsrsOUkLkl58T5f0FG3DTYV8mbSC\n579I/xcTmXV64KDryolcSfuS/gGvJI3fbgj8hLR08Qe2K1+ql9eB7096dwHph+rXJTdr5XIHhwEr\nM/sYcrHt9p0i70JdHniY9EtdfGmepC8A55HeiZ5CSnjfs318qRh6iWkeYN7S8z6SbiDNJYy3vWZu\nu8P2qoXjWAB4hfTzsDtpKPD0kiv9SuvKpA8zd5+uky9vtP33OuOpQ14hcTBpLmEb0troIba/X+j+\nv7T9DUl/IG88aVeiJEVbLB2xNK8T5GHGrzBrKe+VwPGFV5fdYHtdSbe0Jf1bba9eKoa2WEYCreG/\nSbafKnTf/2f7SEnH0vvvRyV7erp1eAfSq/fjpB7u8pKWL7FyRtLZtneRdDu9/0eW3PQxn+2JkpST\n2w8k3QQUSfrM2mzz00L365Pth/MQwsdz0zW2by0Zg6RFSIXOWjFcCRxaw+qq40i7X3+dr/fMbV8o\nGEMnbKBE0i6kYa4rSb39YyV92/a5BW7f+n6LDvt2ZU8/v40+gLQ6Ygppdv6vtivfcShpSduPd0LP\nUtJ1pOGlc0lF1x4DDre9YqkYchyfAS6uow5RWwwHAF9k1sT2DqQaPMcWjOE84A7SUl5IyXZ12/1W\nRa0gjrf0qEv3sjthA2WO41Zgs1bvPpdA+HPhf4u1XLLaq+2u+yCt0JkXmJKvVwLOLxzDEXPSVnEM\na5PGjZcmjSGfB4yp4f/jFNJY+mnAp4FhNcRwG7BA2/UCwG2FY5gyJ20F4riZVASvdf0B4OZC9z4i\n/7lz6e+7j3hu73E9pGdbgRiuIPX6DwVWrfp+3bpO/xXbr0CaqLJ9D1C0dwts1kvbViUDsH2j7X/a\nftT2PrZ3tF16DTJOVT+XB84BdgPuzyubShKzF92bkdtKelmptHMKKNW1f7mf51fl26Sia1dKuor0\nLvDfC937U5IElN4j0ZdLJV0maW9JewMXU/YMb5x2zW8MTAOOl3S70pkPlejW4Z0LSJOW3yAVkXqW\ntG6+8lrdkr5CWmf7QWBq20MLkWrIF6s3ImkCqUf1XL4eDpxpe4tSMfSIZy5S0bN9gA1tL17w3t8i\nVXS8gJTstwN+a7vYmuw8p3AqaYUIpJ/LsbZvKxVDWyzzMKsjdK8LDb1JOoo0zLYgqSihSHNftZwt\nkGPakVQEENJcTy31qXIsHwH+H7Cr7UrKK3dl0m8n6ROkX7JLXaDOR56sG05aKnlg20Mv2H6m6vv3\niGXmyoj+2grEsRWpwudGpAmzs0mHRLxROI61SHMcBv5iu4767a3aKth+XtKOts8rdN+OOVFN0oW2\ntyt1v04m6cOk348dSRvlzgLOc0WriLpy9Y6kxdouWztwi7y6Oa3EmC7paOAZtxVRkrSuKyqi1Ic3\nJS3rXDwqTy7X8Sq/JynRf6lUj7If7T3LWth+vu3yF6S5lhK26ecxU3b39ufyDtg3JX2INO/2Rxda\nNirpBXr/XajjHcfJwJnAFi6wtLwre/qSHgKWIb19FrAoaffjk8AXbVdeAkDpFJy1nP+BJQ0hFXwr\ndliFpC1JRcWuIv07fBzYz/ZlBWPoiGqfkr4P7ExKsCKdbHaO7dKnZ/WM6xHby9QZQx3y0uGPk94V\nXwvcCLxWcvizE+Tfj9Nsf67UPbuyp0/a5n9uK7lJ2pz01ukU0trkEueivqWIkqSi/962L81DGmNy\n0zdcuI69O6fa5+6k5ZGtCf7DSct5a0361PPOqxPI6djMfUk71Y9UOje3UfLvxzKS5i4x/Azdm/TH\n2P5i68L2nyT91PaX8gRWCUWLKPUlJ/mLBnxitVrVPicwexndkqeI/Z20jPeVfD0Pad9C5fraqEd6\nxzGyRAwdSLn+zu6kctMAQ2uMp04PUrDMdLcm/cclfYc0TgZpkuSp/FaqsrMneyhaRKnDdUK1z+nA\nnfmFx6QltZOUS01X/AL06Qr/7rctL2N+daC2ih1AWrZ5ge07JX2AtF69ie7PH60y05Xq1jH9xUk1\nZ1orNa4Ffgg8Dyxre2o/Xx4qoHQw+rJuKzdd+P69Hl/Z4s6pSFo59ThFrK+2UJYKlZnu1p7+xra/\n1t4gaWfb5zD72vnK5BUJxwEjba8qaTVg2xIThz1WL71FDUtHtyHV35kbWE7SGsAhLlhwzal+fa0v\nPHWT9F5gKdIpYmsyawXTwqSTtErG8iHgP4BRtOUhFyiVku/fMat3VLjMdLf29GvvyeSdjt8mVS8s\nWjpW0oPMWpa4LLOvYvqb0wlBxeSVGpsAV5b+t2iLYeYLj+1aXnjqlt/t7A2MZvYiXy+QNqqVXKd/\nK/A/pMN0Zu6ULrGyrtOocJnprurp501AnwKW0uzHAi4MFN0IBMxve1LacT5TkRhaSV3Sb0hjppfk\n661ISxVLe9329B7/FqXmVlp+QCq1fSWA7Sl5HLmIPJ90ap1LEvMQ1riSG8L68Ybt4wZ+Whmq8WD0\nfL9Hevx+zOjrue9UVyV90gqNycC2zH4c3wvANwvH8g9JHyS/hZS0E6nUc0k9VzH9UdKRhWOANIH6\nOWCo0sEuXweuKxxDrS88eWne+0suzevHRfn/YxSzD60cUjCGP0j6N1JZjJkTyDUMPXbCwehFy0x3\nVdK3faukO0g72+qemNuftDFqJUmPkZZl7VE4hr8rFW5qP7KxjsNkvgb8J+mX+wzgMlJFwZI64YXn\nAQouzevHhaTVTDfRlnALa02st59iZ1LFz5JaB6P/2faakjam/O/pl0llppciLSP+ExUeJ9qtY/rX\nAJt2QI+qdRzbkFY5hsL3Xoy0iql1QtLVwA9L96Z6xDSUVOL4+QGfPLj3nZ/0wrN5broM+FFrs1ah\nGA7urd35kPCCcRQ/lrBTSZpse3SeY1gzb6Ks5QSvUro16Z8KfBgo3qNSqubYpxp6dbWT9DtSb2YG\nabv9wsDRto8qdP+hpDru/1HifgMptTSvn/ufABxr+/YBn1xtHKvy1vObTy0cw59J81yHAYuThnjW\ntv2xgjEcSdoZ/jJwKekM5W/a/t9+v/Bf1K319O8n7UJtbXZofZSw0AAflZP0y/znHySN7/lRIoYe\nVs49++2BPwLLkYqwFWF7BrMOqK+NpPUk3QXck69Xl/TrAb6sChsAN0m6V9JtSvXbi5Z3zu96js0f\nGwNHkubiStuOlGy/SUq499N/YboqbJ5/Pz4NPEQ6e+Lb/X7FO9BVY/otrbfLkhbM1/8sfe+adczZ\ntNlceYJqe+BXtl+XVPot5i35Be8cZn/3V3Kn8C+BLUjvQFtzUBv2/yWVKHqYTx92AlYHbrG9j9Lh\n5JX0bPtj+8W2y7rmAVt5eGtSEcCeCw4quVlXyW8bTwMWy9f/APayfWeBex/T3+Ml6s201jrbvqrq\ne82h40k9mFuBq5VKPBcd0ycNITxN2i/QUrqccNGlef3E8LDSCV4r2D5F6VzYBQuH8XIeP39D6XyB\np0iVcYtSOmPgCGAJ0l6WOkorXyTpHtI7jq/k/4/K5pq6MumTVs18y/YVAJI2An4DlBin65jNJXmV\nymG8ddy06AoJ28eQ6hC14vob6S19yRj2KXm/PhRdmteXPLQymnRy1inAXKRe9vr9fd0gmyxpUdLv\n5U2konx/LXj/liOBbWwX/39osX1gHtefnpf2vkQadqpEt07kvmX2va4Z+TqGmNru/RfS6p1fkMYp\n9yGtJPp+6VjCzJpQRwOfJNvgT58AACAASURBVPUo/wQcYPvpwnFMAdYkHYbe2gF6m+3VCt1fwNK2\nH8nXo4CFXc+xkdfaLvliV7tu7ek/IOl7zBrb3oPCZY17DDFJ0jQKDTG1mc/2REmy/TDwg1wSIZJ+\nDXKZ6044JOQ1227Nq+RlxcXke18CfCRfP1Ty/j1MlnQW8Htm3yRWd1XYynRr0v88qapm6z/umtxW\nUp1DTC2vKp3YdZ+kr5I2fhQdu833H2O79EaonjHsZPvsumLIcSxH2qg2itl3wpZetXK2pOOBRSV9\nkfS78ZvCMdwsaW3bNxa+b08Lkw5o37ytrfhcT0ldObzTCTphiEnS2qQx40VJOw8XAY60fX2pGHIc\nxQ9j7yWGybZH1xzDraRqirfTVgKijgl3SZuREp2Ay2xPKHz/e0hLEx8mraZqTaAWGWLqBEqn2vXJ\n9s2V3Lebkv5Aa9BL9qgkXQDczOxDTB+1vUOpGNpiWZj0C1V8V3C+/09Jk3Tnu6YfOKXjEf8BnMXs\nSzaL7U6WdIPtEkd1dry8gust8jBkyTiWJu0VaI3rX0OaZ3m0wL37OzTGrqjMdLcl/WnAI6T6Ljcw\nq144ULZHJWk4aYipdZDLNaQSCM8WjGE0aXVGa1PYdODzpcvXKtUuX4C0PPFl6qlZ/mAvzS65kinX\n/lmBNIHbPn5cSY+ul/v3VUO+FUfJ/4/TbO85UFuBOCYAv2P2ztnutjcrGUdJ3Zb0h5KOwduNtJX5\nYuCMwpOnHSPvstzf9jX5egPSIdSNeQvdSSQdRtqJfD+zhncq69H1E8ehpIqvp5FegHcHliy5qks9\nzrfIv7u32165VAz5vlNsrzFQW4E4ypWksN2VH6SDr/cGpgFfreH+E4BF266Hk8ZOS8ZwSy9tN9f0\n/7EtaYfwT4FP13D/+UnnFZ+Qr1coHQfp1La56/j37xHHrXPSVtG9DyKVOn+DtEHv+Xz9NHBYDf8W\nE0m9+6H5Yw9gYuEYDiadD/wk6Z35E8C5Vd2v62rvSJon77L7X1J50mNINbtLW9z2c60Lp2GdJQrH\ncJWk4yVtJOkTuc7LlZLWGmgSaTDl8fQDgLvyxwG511vSKcBrzFo99RipyFVJd5Am1ev2oqTdJQ2V\nNETS7rTNc1TJ9mG2FwKOsr1w/ljI9ntsH1Qihh4+D+xCSrRPkMpDlN7ItxOwKfCE0ybC1UmLLirR\nbcM7pwKrApcAZ9q+o8ZYbgJ2cD6BJ09cXeCyRzbWMlHUSxy3AWvYfjNfDyW9Cyk2zKRZJXRnriSq\nYTXVlaRhxxuZfUy/6JLNvBnqaNLkpYFrgW+43vXyjSVpku11cs7YmPTO527bK1Vxv25bp78Hqcdy\nAPD1thonddTT+E/gL0pn5Qr4OLBfwftju2ipgwEsCrRWylTWi+nHa0oHo7c2JH2Q8geI9FpPv7Sc\n3Cvb5v9uosJljftQtCRFV/X0O03edj8mX17vtCOz5P1HAj8B3md7K0krA+vZPqlwHLsBh5PGLUU6\n1OVA22cVjGEz0pj+yqTVM+sDe9u+slQMnULSKfSyisd26Q2MtWtN2kragVTa+FvA1SXfAfaIZxQV\nl6SIpN/FJP2RNJb9n7ZXlzSMNKzykRpiWRJYO19Osv1EDTG8h/QiLOp5EW5fMjk3qdDZi4XfgSJp\nx7bLeYEdgL+7QAXYthh+BpzsmlfWSbrT9iqSTiRNnl5aathP0kq27+lrfs0VLeXttuGdMLvFbZ8t\n6SAA229IKl7KN9/7cXId+RotRVqhMQzYUBIuWGMlT2ACM4uObcesd4LF2D6v/VrSGcBfCodxN3BC\n7oicQlpaPb1wDADjVbCscQ/fIg35/qyXx8zsZcAHTfT0u1ieONwRmGB7LUljSMcGfqLeyMqTdDJp\nvPZOZl8jX+uQRoeUqFgRuNj28jXdex/S3pprgd8416sqcO8hpBfde5hV1ngBYKE63omWEj39inTI\njsNvkXrXH5R0LTCCtDysica48MafnvJS4pYhpJr2xQ5mb4uj587cJ4Dv1BDHUGCl/PEP0iE735L0\nJdufrfr+Toe4/Hf7i67TSVpFlq+2SNofOL21xDvv5t/NdiVHaUbSr84q7Rf5B/yjpW6eezHzAp8g\nHZYh4F7br5eKoS2WTngB/KuklW3fVfCePbWfvfoG6TSx4qto2oeZ6iKpdcbDROAntiflh46QdG/B\nUCbmOY7a6kIBX7T9360L288qVT+tJOnH8M4gy+Pn3wXmI5Vsba0bfY20G7TYBpROGDrIcdS+5V7S\nJ0jvep4gLdVsXFXHFkkTbW86UFvFMewDnO3Zz6htPbZIqfH9DqkLdTuwWutFJ/9+3GZ7lf6/8l/T\ndTty69bLjsOFatxxOFHSjlKFpyz3Q9JB+ZdqNUnP548XSOehXlg4nJNIdW+2JPUwP83sPe/KSTpS\n0sKS5pI0UdI0SXsUvP+8khYDFpc0XNJi+WMUaZK7pHHADpK+n2NbVtI6ACUndPPv5hDbc7X9vhZd\nTUXaH3CWpE0lbUoqGHlpVTeLnn5FcqLdgbYqm7Z/XziG2nsxOY7Datpi3x7DX22vV3MMta4Jl3QA\n8A3gfaQyFK3OwPOkCdRflYgjx3IcaUJ9E9sfzuPYf7K99gBfOthxtArOLWf7UEnLkIrPTRrgSwcz\nhiHAl0ilGCDV7TrRdiUr7SLpV0Spzs3ypFdtgF2B+23vX19U9ZG0FPB+Zj8x6uqC9/81aVfwH6jp\nWDxJd9hetY414T3i+JrtY0ves5cYbs4rymori5Hv2REvPiXFRG51NgE+3DZON460XLCovGKktncb\nOYbDgc+Siq21ei8GiiV90hzLq9R7LN5FNa4Jb52k9kgr4Uvai7Sk92HgBy54oAzweh67bv1+jKDt\nNLGC1m29+MDMSdS5SwYgaX3gB8zqFLXekVdy1kMk/epMBZYl/UIBLJPbiunl3caXJW1Ww7uNHYAV\nbZeudTOTU/XCWtk+UKnWS2tN+IuUXb1zPPBJAEkbkkpjfA1Yg3Smc8nlvK3qt0tI+nG+938VvH9L\nJ7z4nAR8k1R3p/LNk5H0B5mkP5B+gBYC7pY0KV+vCxQbJ8w64t0G8ACp5EBtSV+dU29mJWBU3ona\nUs1hGW81tK03vytpNdl5wHmSphSKAQDbpytVldyU1LPd3vbdJWPIWi8+I2t88Zlu+4+lbhZJf/D9\ntO4A2tT+biN7CZgiaSKzj6cXq/UCXNT2+cx6MwXvj6TTgA8CU5h9mKtY0pc0zPYbpGTbXvW1jlxw\nH2kSeRikFTzOpchL6fHiA/W8+Fwh6SjSUGPlx2hG0h9kbjuHV6mG/gq2/6xU1rf0v3fPdxvrkMq4\njs+xlqrjPp6a6+50SL2Z0cDKNW4COoN0sM4/SPMKrWM0lyedn1yMpK+RSk0/SXoBFOlntI59E/OT\najKZNPdT2rr5z9FtbVF7590m76jbD1jM9gclrQD8T+ENMP3W2HHBg+I7jWqoNyPpHODrTsXnaqFU\nf2lJ0gqVF3Pbh4AFq+pZ9hHHVNIk6tOl7tlHHN8HdgbOIw8zAefYLn2qWjGR9CuSx0jXAW5oW5J2\nu2soa1w3SQ/S+3h6JasT+oiht3ozB/V8B1BxDFeQJk0nUePJWZ0g/1tsloea6ozjXmB126/k6/mA\nKbZXLBzH1qTSLe0Hox9Sxb1ieKc6r9p+rbUZNk/cNfUVtv1t67ykntViJQPohHozpGV5IXmAdF7z\nxcz+AvjzwnH8nfQz2Vo6Ow9p41oxkv6HNMS0MXAiaTK5skUfUYahOldJ+i4wn9KpTeeQNgY1ju2n\n2z4es/1LYOuSMUhaX6lsLpL2kPTzPOdSTB5Ou4c017IQ6RzUpg6x/Y2083RuZv171PHCPB24U9Jv\n8wqvO4DnJB0j6ZhCMXzM9l7As7Z/CKwHfKiqm8XwTkXy1up9SZuBBFxG2lpd5B88rz0+1fbuJe43\nQCztJwO1Sgp/peTuS6XD2VcnTRT+ltSj2sUFzxaQtAtwFHAlzDw3+du2zy0VQ6eRtCCA7X/WdP+x\n/T1ue1yBGG6wva6k64HPAE8Dd1Y13xTDOxVxqtX9e+D3tqfVcP8Zkt4vaW7br5W+fw/tJwO1Sgrv\nUjiGN2xb0nbAr2yfJGnfwjH8J7C27adg5kagPwONS/qSVgVOIw/z5RVFe7nw8YklkvocuEjpYPSj\ngJtJw8AnVnWz6OkPslzA6WDgq8waPpsBHFvVxEw/sZwKfJi0XHJmCdsaxk1rJ+kqUuXCfUgHsz8F\n3FpyYr3nRH5+N1g0hk4h6TrS2c1X5OuNSHX1P1ZrYDWTNA8wryusNBo9/cH3TWB9Uo/uQQBJHwCO\nk/RN278oGMv9+WMI9YyXAqk+OumFcMPcdBVwSJU/2L3YFfgcsK/tJyQtS+pZlXSppMuYvQhfsZ2Y\nHWYBtx2LaPvK1pxLE0n6GDCKWRvVsF3Jpr3o6Q+yXLhpM9v/6NE+grQ2uvihJh0wbnoeaYKs9VZ6\nT9Iyuc/0/VXdqa0AHqQCeBfUGU9dJF1AGso4LTftAXzU9g4FYxhKOjP6P0rds484et2pXdWO9Uj6\ng0y5fO7bfayiWGYbNyWdQ1p83FS5jvxAbRXH8BngCGAJ0iRqsbMF8o7Xkbav7dG+AfC47furjqHT\nKJUw/iFtL4CkSp/PFo7jettjSt6zlxjupuBO7RjeGXz9TZqWnlA9AfhWj3HT3wClx01flrSB7b/k\nONYnlQEo6Uhgm5qKev0S6O0Qmen5saIneHWCnNxL1l7qyy25LMk5zD7vVbLk9h3Ae4EiO7Uj6Q++\n1SU930u7aNttV0injJt+BRiXx/YBngX2LhzDkzUlfEi9/Nt7Ntq+XemowsZo1X3qSw27k+clLZFs\nr3NT+pyFxYG7co2syndqR9IfZLaH1h1DmwckfY/Zx00fKB2E7SmkF8OF83VvL4pVmyzpLOD3lD85\na9F+HqujwFed1gMeIU1m38CsIxtr4Q44Z4HCO7VjTL+L9Rg3NWnc9Ic1jJv+BDjS9nNtcf277WJ1\ny/Nuy57sAvX0c0XPy23/pkf7F0iT/rtWHUOnyJOnmwG7kTbKXQycUXqeqS2epYFjSSvuIP2OHGD7\n0TriKSGSfqic2s5BbWu72fZafX1NN5E0knRQx2uk05Eg7UqeG9jB9hN1xVanvCZ9N9LS2R+64MHs\nbTFMAH7H7O+Gd7e9WYF7/8X2Br0UA6x0kUEk/VC5XAJhbefjEnMlw8m2VykYQ+09OkkbA63VW3fa\nvrzUvTtJTvZbkxL+KNLmwZNtFy10lmOpbWWZpA/YLj7cGmP6oYTTgYltQyz7MGvNfimnkHp0O+fr\nPXJb5T26ljypfsWAT+xieZf4qsAlpN79HTWH9LSkPZi1YW430sRuCecAH5U00SXP2YiefihB0pbk\nQ7mBCbYvK3z/2vcKBJD0JrOWRhYb0ugnnveT3gGul+O5jnTQTeXHNuaNnOeQVre9Zad+VeVSoqff\nhSQdSz+1+6va6dcXScsBV9q+NF/PJ2mU7YcKhlFnjy5ktjuqnLvth4G6DrH5LOmkrmEULJMSPf0u\n1FYudn1gZeCsfL0zcJftLxeOZzKpZvhr+Xpu4FrbaxeMobYeXQj9kbSV7WI1mCLpd7Fcn3sD5yPp\nJM1FqvdSdNt5H0Mrt5aspx9CSDrqrVYYdMOB9jHSBXNbadMkzXwLnWva/6Of5w86SeNyzfLW9XBJ\nJ5eMIYROEGP63e1wUm2RK0gTZRtSzzmtXwZOl/SrHMcjwF6FY1ittTkMUu0XScUrnobOljskT9i+\nodD9hgBjbF9X4n4QwztdKx/msjTwOrBubr6hzo1AdZZ4lnQrsFFrN7KkxYCrmniASehb3j3+EWCY\n7a0K3fMtmxcrvV8k/e7V86SmOknaGliFtqJzJU8Sk7QX8F3SEjlIk9o/tn1a318VQvUk/RT4K3B+\nifLKkfS7mKRxpPNgb6w5jv8B5gc2Jp39uRMwyXbRM2olrcysaoqX276r5P1DZ+p5ahVQ2alVfdz/\nBWAB0gEqLxNlGMK/StI9wPLAw6QNMa0fptUKx3Gb7dXa/lwQ+KPtj5eMI4SeSp9a1QliIre7bVF3\nAFnrwJSXJL2PtClqyRrjCaFlNAVPrepNnn/bHVjO9qGSlgGWtD2pivvFks0uZvvhvOPwZdKGpNZH\naRfl5ZJHkc5FfYhUByeEurVOrarTr0mbBj+Xr/8J/HdVN4vhnS6W18b/DHgf8BTwfuDuktUte4lp\nHmBe29PriiGElryceQ2gyKlVfcRws+212lfxVLl5MYZ3utuhwBjgz7bXzKV996gzoFxe+dUBnzhI\neqlVPvMhaijwFTrOD+oOAHg9Hy5jAEkjgDerulkk/e72uu2nJQ2RNMT2FZJ+WXdQJdkuVsgqvPvY\nvqruGIBjSIfsLCHpx6TVbZWdKhdJv7s9l1fKXE3aEfsUs8raNpKkJZh9r0AUXGswSWNIhfg+TDrJ\nbCjwYsl3gLZPl3QTsCnpHej2tu+u6n4xkdvdtgNeAr4JXArcD2xTOghJh/S4Hirp9MIxbCvpPuBB\n4CrSZHKxyoahY/2KVGb7PtIh9V+gwknUftxH6u2PB16UtGxVN4qk392WAOa2/YbtccBvKFi3u80y\nkg6CmRO555N+yEtqzW/8n+3lSL2q6wvHEDqQ7anAUNszbJ8CbFny/pK+BjwJTAAuIh0Wf1FV94vh\nne52DvCxtusZua1YHfvs86ThpYNIu3IvsV16bqHx8xuhVy/l8x2mSDoSeJzyneEDgBVtFznUJ3r6\n3W1Y6+ASgPz53KVuLmktSWsBawJHA7uSevhX5/aSes5vHE3D5zcCAHuS8uBXST8PywA7Fo7hEaDY\nEuZYp9/FJE0AjrU9Pl9vRzotqsghzHkNdF9se5N+Hh/sWBYAXiFNlO0OLAKcXqp3FTpX7ul/KF/e\na/v1Qvf9Vv50FWBF0rBO+16BSs7IjaTfxSR9EDidtDkL4FFgT9v31xdVCJ1D0kbAONLEvkg9/bG2\nry5w74P7edhVVaGNpN8Addaxz/efh/SWeRSzVzIsWVr5M8ARpMltEZuzApCXSn7O9r35+kPAGbY/\nWjCGnW2fM1DbYIkx/Qaw/c+6En52IWn56BukcdPWR0lHAtvaXsT2wrYXioQfgLlaCR/A9v8BcxWO\n4aA5bBsUsXonlLC07aLL4HrxZJUbXsK71mRJJwL/m693ByaXuLGkrYBPAUtJOqbtoYVJHaRKRNLv\nYpLmybVu+m0r4DpJH7F9e+H7tpss6Szg98w+WXZ+fSGFDvAVYH+gVT//GlLVyxL+DtwEbJv/bHmB\ntKGyEjGm38Va1fsGaisQx12kw1weJCXc4oe5SDqll2bb/nypGELoTZ5zG5Uvp9p+pcr7RU+/C0l6\nL7AUMJ+kNUlJFtLbxvlrCKnIAdP9sb1P3TGEziHpdvo5W6JEh0TSMOAnwD7A38irh3IH5T+rWjoa\nSb87bQHsDSwNtK/1fYF0OHhR+SCXtxQ7K0nS0qTCWuvnpmuAA2w/Wkc8oXafrjsA0qFCCwEfsP0C\ngKSFgZ/mjwOquGkM73QxSTvaPq8D4qj9MJe8Ue13wGm5aQ9gd9ublYohhHa5AOCHeh7VmGvr32N7\nhSruGz39LiRpD9v/C4xq2/U3U1U7/frRCYe5jMjFtFp+K+kbhWMIoZ17O5vX9gxJlfXGY51+d1og\n/7kg6e1jz4/SXs/lDmYWOyMdSF3S05L2yGWdh0rag3RAewh1uUvSXj0b88/mPVXdNIZ3QuUk/RnY\nHjgMWJw0xLO27Y/1+4WDG8P7SWP66+Wma0l1iOIQlYaTNB+wbPsmrUL3XYpUZvxlZi3ZHE2q67+D\n7ccquW8k/e7TY6PHW9j+en+PD7Zc7Oxl0jvLKHYWOoakbUiTpnPbXk7SGsAhhQ9G34RUdA3gLtsT\nK71fJP3uI2ls/nR9YGXgrHy9M+mH6su1BAZIWhx4urexzIrveyTwI9KLz6XAasA389xHaKhce2cT\n4Erba+a2221/pN7IqhNj+l3I9rh8UtZqwEa2j7V9LOm0qDVKxSFpjKQrJZ0vaU1JdwB3AE9KKl2W\nYXPbz5OW6j1E2iz27cIxhM7zuu2etey7uiccq3e623DShqxn8vWCua2UX5H2BSwCXA5sZft6SSsB\nZ5B63KW0imhtDZxje7qk/p4fmuFOSZ8DhkpagVSO4bqaY6pU9PS72+HALZJ+K2kccDNpB2Apw2z/\nKZeIfcL29QC2K1uZ0I/xku4BPgpMlDSCdKhKaLavkcbTXyV1RJ4Hunopb4zpd7lckmHdfHmD7ScK\n3ntmnZ+eNX9K1gCSNIS0T+AeYHpeB70AsFDJf48QOkEk/S6mNH6xO2mb9yGSlgXea3tSofvPINXN\nF2kZ2kuth4B5bRerWy7pltZEXQiS/kD/tXeKrd4pLZJ+F5N0HPAmsIntD0saDvzJ9to1h1acpJ8C\nfwXOL71yKHQeSZ/o73HbV5WKpbRI+l2sNYTS3suVdKvt1euOrTRJL5B2Ks8gLduM4xIDkg6wffRA\nbd0kJnK72+u5eJMB8uTlm/WGVI98POIQ23PFcYmhzdhe2vYuHURJsWSzux0DXAAsIenHwE7Af9Ub\nUj3a5jeWs32opGWAJUvNb4TOImk34HPAcpLGtz20ELOWOHelGN7pcnlN/Kak4YyJTT0nNuY3Qrtc\ni2k5Uj2oA9seegG4zXZlZ9TWLZJ+l8rDOnfaXqnuWDpBzG+EkMSYfpeyPQO4Ny/TDDG/EdpI+kv+\n8wVJz7d9vCDp+brjq1KM6Xe34aRt5pNI6+WB7l6D3I/W/MbIps9vBLC9Qf6zjvMlahXDO12sr7XI\n3bwGuT9t8xsAlzd1fiPMIuk023sO1NZNoqffhSQtD4zsmdwlbQA8Xk9UHWF+oDXEM1/NsYTOMNs5\nzZKGkeozda0Y0+9OvyQVjuppen6scSR9HxgHLEY6vesUSTG801CSDsob9lZrH88HngQurDm8SsXw\nTheSdGNfSxG7/YCIvki6F1jd9iv5ej5giu0V640s1EnSYbYPqjuOkqKn350W7eexpg5r/B2Yt+16\nHqCSM0jDu0fTEj5E0u9WkyV9sWejpC8w6wDmpplOWsn0W0mnkE7wek7SMQOdKRxCN4nhnS4kaSRp\neeJrzEryo4G5gR2aWEO+7dzgXuXjJUPoepH0u5ikjYFV8+Wdti+vM54QQv0i6YcQQiaptXfjv23/\nqtZgKhLr9EMIIcvF+BZn1hGjXScmckPXkzQ0n5wVwoBs/8P2xXXHUZVI+qHr5eJzG9QdR+g8kj4j\n6T5J05tScC3G9EMj5Hr6SwHnMHvxufNrCyrUTtJUYJsm1WGKMf3QFPMCTwObtLUZiKTfbE82KeFD\n9PRDCA0m6WjgvcDvgVdb7d38DjB6+qERJH0IOI5UfXRVSasB29r+Uc2hhXotDLwEbN7W1tXvAKOn\nHxpB0lXAt4Hj245LvMP2qv1/ZQjdJXr6oSnmtz1JUntb1x5+HeaMpHmBfUl19WcW5LP9+dqCqlgs\n2QxN8Q9JH2TWGbk70ewDZUJyGmlMfwvgKmBp4IVaI6pYDO+ERpD0AeAE4GPAs8CDwB62H6ozrlAv\nSbfYXlPSbbZXkzQXcI3tMXXHVpUY3gmNYPsB4JOSFgCG2O7q3lyYY6/nP5+TtCrwBLBEjfFULpJ+\naARJ8wA7AqOAYa2xfduH1BhWqN8JkoYD3wPGAwvmz7tWDO+ERpB0KekglZuAGa122z+rLagQahBJ\nPzRCLM8MvZG0CPAD4OO56UrgUNvT64qparF6JzTFdZIadyB8GNDJwPPALvnjBeCUWiOqWPT0Q1eT\ndDtpmeYwYAXgAdJ2ewG2vVqN4YWaSZpie42B2rpJTOSGbvfpugMIHe1lSRvY/guApPWBl2uOqVLR\n0w+NIOk023sO1BaaRdIawDhgEdK7v2eAvW3fWmtgFYqefmiKVdovJA0FPlpTLKFD2J4CrC5p4Xzd\n1QeoQCT90OUkHQR8F5gvn4jUKr7zGmmHbmggSd/qox0A2z8vGlBBkfRDV7N9GHCYpMNsH1R3PKFj\nLFR3AHWJMf3QGJK2BTbMl1favqjOeEKoQyT90AiSDgPWAU7PTbsBN9r+bn1RhVBeJP3QCJJuA9aw\n/Wa+HgrcEuv0Q9PEjtzQJIu2fb5IbVGEjpFf/BslJnJDUxwG3CLpCtIKng2BA+sNKXSA+ySdB5xi\n+666gykhhndCY0haElg7X06y/USd8YT6SVoI+CywD2nk42TgzG5erx9JP4QQAEmfAH5HGgY8l1Rt\nc2q9UQ2+GNMPITSWpKGStpV0AfBL4GfAB4A/AJfUGlxFYkw/hNBk9wFXAEfZvq6t/VxJG/bxNe9q\nMbwTul5eoXGn7ZXqjiV0FkkL2v5n3XGUFD390PVsz5B0r6Rlbf+t7nhCR3lD0v6kgnzzthptf76+\nkKoVY/qhKYYDd0qaKGl866PuoELtTgPeC2wBXAUsTTo9q2vF8E5ohLwy4y1sX1U6ltA5JN1ie01J\nt9leTdJcwDW2x9QdW1VieCc0gu2rJI1k9nX6T9UZU+gIr+c/n5O0KvAEsESN8VQuhndCI0jaBZgE\n7Ew6APsGSTvVG1XoACdIGg58DxgP3AUcWW9I1YrhndAIkm4FNmv17iWNAP5se/V6IwuhrBjeCU0x\npMdwztPEO93G6uvkrJY4OSuEd79LJV0GnJGvd6VLd1yGOdI6OWtF0jxPayXXNqRhwK4VwzuhMSR9\nBtggX15j+4I64wn1k3Q1sLXtF/L1QsDFtrtyNy5E0g8hNJike4HVbL+ar+cBbrO9Yr2RVSeGd0II\nTXYqMCkXXAPYHvhtfeFUL3r6IYRGk7QW8PF8ebXtW+qMp2qR9EMIjSXpZ8DJtu+sO5ZSYslaaCRJ\n4yQdl3dhhua6m7RB6wZJX5bU9WcnR08/NJKktYFlgXVsf6fueEK9JK1IOjJxN+Ba4De2r6g3qmpE\n0g+NI2kIsGA3n4Ma0V2TDwAADhFJREFU5lw+b+HTpKS/DHA2aWnvi7Y/W2dsVYjhndAIkn4naWFJ\nCwB3AHdJ+nbdcYV6SfoFcA/wKeAntj9q+wjb2wBr1htdNSLph6ZYOffstwf+CCwH7FlvSKED3Aas\nYftLtnvuxF2njoCqFuv0Q1PMlWulbw/8yvbrkmJss+FsnyJpuKSeJ2ddbXt6jaFVJpJ+aIrjgYeA\nW4GrJb0fiDH9hpP0BeAA0olZU4AxwF+BTeqMq0oxkRsaS9Iw22/UHUeoj6TbSQXXrre9hqSVSGP7\nn6k5tMrEmH5oBEkjJZ0k6Y/5emVgbM1hhfq9YvsVSHV3bN9DqrzZtSLph6b4LXAZ8L58/X/AN2qL\nJnSKRyUtCvwemCDpQuDhmmOqVAzvhEaQdKPttVsHYee2KbbXqDu20BkkfQJYBLjU9mt1x1OVmMgN\nTfGipPcABpA0BujK1RlhYJLmBb4MLA/cDpxk+6p6oyojevqhEXIlxWOBVUmbs0YAO9m+rdbAQi0k\nnQW8DlwDbAU8bPuAeqMqI5J+aAxJw0iTdALutf16zSGFmki63fZH8ufDgEm216o5rCJiIjc0gqT9\nSfV27rR9B7CgpH+rO65Qm5kv+E1bths9/dAIvU3atk/qhmaRNAN4sXUJzAe8lD+37YXriq1qMZEb\nmmKoJDn3cnJlxblrjinUxPbQumOoSyT90BSXAmdJOj5ffym3hdAoMbwTGiHX0P8SsGlumgCcaHtG\nfVGFUF4k/RBCaJAY3gldTdLZtnfJhbXe0sOxvVoNYYVQm+jph64maUnbj+dSym9hu6vrrITQU/T0\nQ1ez/Xj+dEfgTNt/rzOeEOoWm7NCUyxEqqJ4jaSvShpZd0Ah1CGGd0KjSFoN2JXU83/U9idrDimE\noqKnH5rm/7d3d7GeVWcdx7+/GaC8w7S2OIJgqBasOlBCSwXElEKbasSAMBjQxNiikUaq8UYbiaZG\nRE0EubBWvKpNDFRb0UZLeZFKrXTKO05xgkqwCdQGaGAELXR4vPjvkx7GI/Rqr8l+vp9k55y99s3v\nYuaZNc9ee62vAl8BngLeMDiLNDuLvlpIcnmSO4DbgNcBl7lyRx35IlddHAP8UlXdPzqINJI9fS3e\ntM/Ozqo6cXQWaTTbO1q8aauFXUmOHZ1FGs32jrrYAuxMsoNvbqlLVZ03LpI0P4u+urhydABpX2BP\nX21MWzF8T1XdmuRgYHNV7R6dS5qTPX21kOQy4C+Atf30jwb+alwiaQyLvrp4P3AG8CxAVT2CH2ep\nIYu+uvh6Vb2wdpNkPzbYallaOou+uvhskg8CByU5F/g48DeDM0mz80WuWpiOS3wv8C4gwM2sjkv0\nL4BaseirnSSvBY6pqgdHZ5HmZntHLSS5I8nhU8G/B7g+yTWjc0lzs+iriyOq6lngAuCjVXUa8M7B\nmaTZWfTVxX5JtgLbgU+NDiONYtFXFx9i9fL2X6vqi0mOBx4ZnEmanS9yJakRZ/qS1IhFX5Iasehr\n8ZJsSrJ9dA5pX2BPXy0kubuqTh2dQxrNoq8WklwNPAncwMtPznp6WChpAIu+Wkjy6AbDVVXHzx5G\nGsiiL0mNeEauFi3J2VV1e5ILNnpeVZ+YO5M0kkVfS/fDwO3Aj23wrACLvlqxvSNJjTjTVxtJfhT4\nPuDAtbGq+tC4RNL8/DhLLST5Y+Bi4BdZnZx1EXDc0FDSALZ31EKSB6tq27qfhwJ/V1U/NDqbNCdn\n+uriv6efzyf5DuBFYOvAPNIQ9vTVxaeSHAn8PnAvq5U714+NJM3P9o7aSfIa4MCqemZ0FmluFn21\nkORA4HLgTFaz/M8BH66q/xkaTJqZRV8tJLkR2A18bBq6BDiyqi4al0qan0VfLST5UlW9+dXGpKVz\n9Y66uDfJ29dukpwG3D0wjzSEM321kORh4ATgP6ahY4FdwDdYbbG8bVQ2aU4WfbWQ5BW/vq2qx+bK\nIo1ke0ctTEX9O4Gzp9+fAzZV1WMWfHXiTF8tJPkN4FTghKp60/RV7ser6ozB0aRZOdNXF+cD5zGd\nj1tVjwOHDU0kDWDRVxcv1Oq/tQWQ5JDBeaQhLPrq4sYkHwGOTHIZcCvuvaOG7OmrjSTnAu9itZ/+\nzVV1y+BI0uws+lq8JJuBW6vqHaOzSKPZ3tHiVdUe4KUkR4zOIo3mfvrq4r+Ah5LcwrSCB6CqrhgX\nSZqfRV9dfGK6pNbs6UtSI/b0JakRi74kNWLRVwtJ/s8JWRuNSUtnT18tJLm3qk55tTFp6Vy9o0VL\n8h7gR4Cjk1y37tHhrA5QkVqx6GvpHmd1LOJ5wD3rxncDvzwkkTSQ7R21kGS/qnJmr/Ys+lq0JDdW\n1fYkDzFtq7yeZ+OqG4u+Fi3J1qp64v87I9ejEtWNRV+SGvFFrhYtyW42aOusqarDZ4wjDWfR16JV\n1WEASX4LeAL4M1aHqFwKbB0YTRrC9o5aSPJAVZ30amPS0rkNg7p4LsmlSTYn2ZTkUtbtqy91YdFX\nF5cA24H/nK6LpjGpFds7ktSIM321kORNSW5L8s/T/bYkvz46lzQ3i766uB74NeBFgKp6EPjJoYmk\nASz66uLgqtqx15h78agdi766eDLJG5k+1EpyIat1+1IrvshVC0mOB/4EOB34GvAocKl776gbv8jV\n4iXZBJxaVeckOQTYVFW7R+eSRnCmrxaS3F1Vp47OIY1m0VcLSa4GngRuYN2XuFX19LBQ0gAWfbWQ\n5NENhquqjp89jDSQRV+SGnHJplpI8v4kR66735Lk8pGZpBGc6auFJPdX1cl7jd1XVW8ZlUkawZm+\nuticJGs3STYDBwzMIw3hOn118WnghiQfme5/fhqTWrG9oxamD7R+DjhnGroF+NOq2jMulTQ/i74k\nNWJPX5IasehLUiMWfUlqxNU7ainJVcAzrF7mPjU6jzQXZ/rqagerk7OuGR1EmpOrdySpEds7aiHJ\ndRsMPwPcXVU3zZ1HGsX2jro4EDgZeGS6tgHHAO9Ncu3IYNKcbO+ohSR3AWesfYGbZD/gTuBM4KGq\nevPIfNJcnOmriy3AoevuDwFeO/0j8PUxkaT52dNXF78H3J/kDiDAWcBV00Hpt44MJs3J9o7aSLIV\neNt0+8WqenxkHmkEi77aSHI0cBzr/odbVf8wLpE0P9s7aiHJ7wIXAzuBl6bhAiz6asWZvlpIsgvY\nVlW+tFVrrt5RF/8O7D86hDSa7R118Tyr1Tu3sW6JZlVdMS6SND+Lvrr46+mSWrOnL0mNONPXoiW5\nsaq2J3mI1Wqdl6mqbQNiScM409eiJdlaVU8kOW6j51X12NyZpJFcvaNFq6onpl8vr6rH1l/A5SOz\nSSNY9NXFuRuMvWf2FNJg9vS1aEl+gdWM/o1JHlz36DDgH8ekksaxp69FS3IEq22Vfwf41XWPdlfV\n02NSSeNY9LV4STYDO6vqxNFZpNHs6WvxpoNSdiU5dnQWaTR7+upiC7AzyQ7gubXBqjpvXCRpfhZ9\ndXHl6ADSvsCevtpIchTw1ul2R1V9dWQeaQR7+mohyXZgB3ARsB34QpILx6aS5udMXy0keQA4d212\nn+T1wK1VddLYZNK8nOmri017tXOewj//asgXueri00luBv58ur8Y+NuBeaQhbO+ojSQXAGdOt3dW\n1SdH5pFGcKavTj4P7AFeAr44OIs0hD1NtZDkfaxW75wPXAjcleRnx6aS5md7Ry0k2QWcXlVPTfev\nAz5fVSeMTSbNy5m+ungK2L3ufvc0JrXiTF8tJPko8APATazOyv1x4MHpoqr+YFw6aT6+yFUX/zZd\na26afh42IIs0jDN9tZLk4Kp6fnQOaRR7+mohyQ8m+RLwL9P9SUn+aHAsaXYWfXVxLfBuppe3VfUA\ncNbQRNIAFn21UVVf3mtoz5Ag0kC+yFUXX05yOlBJ9gc+ADw8OJM0O1/kqoUk3wb8IXAOEOAzwAfW\nPtaSunCmr8VLshn46aq6dHQWaTR7+lq8qtoDXDI6h7QvsL2jFpJcA+wP3AA8tzZeVfcOCyUNYNFX\nC0n+foPhqqqzZw8jDWTRl6RG7OlLUiMWfUlqxKKvFpK85lsZk5bOoq8u/ulbHJMWzY+ztGhJvh04\nGjgoyVtYfY0LcDhw8LBg0iAWfS3du4GfAY4B1p+OtRv44IhA0kgu2VQLSX6iqv5ydA5pNIu+Fi3J\nT1XVx5L8CquzcV/Gs3HVje0dLd0h089Dh6aQ9hHO9CWpEWf6WrQk173S86q6Yq4s0r7Adfpaunum\n60DgFOCR6ToZOGBgLmkI2ztqIcldwJlV9Y3pfn/gzqp6+9hk0ryc6auLLaw+yFpz6DQmtWJPX11c\nDdw37asf4CzgN4cmkgawvaM2pi0ZTptuv1BVXxmZRxrB9o5aSBLgHOCkqroJOCDJ2wbHkmbnTF8t\nJPkw8BJwdlV9b5ItwGeq6q2Do0mzsqevLk6rqlOS3AdQVV9L4pJNtWN7R128mGQz0/47SV7PauYv\ntWLRVxfXAZ8E3pDkt4HPAVeNjSTNz56+2khyIvBOVks2b6uqhwdHkmZn0dfiTW2dnVV14ugs0mi2\nd7R4VbUH2JXk2NFZpNFcvaMutgA7k+wAnlsbrKrzxkWS5mfRVxdXjg4g7Qss+lq0JN8NHFVVn91r\n/EzgiTGppHHs6WvprgWe3WD8memZ1IpFX0t3VFU9tPfgNPZd88eRxrLoa+mOfIVnB82WQtpHWPS1\ndHcnuWzvwSTvY3WMotSKH2dp0ZIcxWr7hRf4ZpE/ldX5uOe7p766seirhSTvAL5/ut1ZVbePzCON\nYtGXpEbs6UtSIxZ9SWrEoi9JjVj0JakRi74kNfK/MEnE1dC9FPIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CXWBl1Ao4FW4",
        "colab_type": "code",
        "outputId": "46ba5d89-7497-4f0a-f89c-6f25f9cc85fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train_raw=train_raw.rename(columns = {'consumer_complaint_narrative':'text', 'product':'label'})\n",
        "train_raw.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>In XX/XX/XXXX my wages that I earned at my job...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>XXXX was submitted XX/XX/XXXX. At the time I s...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I spoke to XXXX of green tree representatives ...</td>\n",
              "      <td>Mortgage</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>i opened XXXX Bank of America credit cards 15-...</td>\n",
              "      <td>Credit card or prepaid card</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I applied for a loan with XXXX XXXX and had pu...</td>\n",
              "      <td>Consumer Loan</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text                        label\n",
              "0  In XX/XX/XXXX my wages that I earned at my job...                     Mortgage\n",
              "1  XXXX was submitted XX/XX/XXXX. At the time I s...                     Mortgage\n",
              "2  I spoke to XXXX of green tree representatives ...                     Mortgage\n",
              "3  i opened XXXX Bank of America credit cards 15-...  Credit card or prepaid card\n",
              "4  I applied for a loan with XXXX XXXX and had pu...                Consumer Loan"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JFckqSM_4Pvr",
        "colab_type": "code",
        "outputId": "46f83ccc-0d07-406c-a5d7-783c4b308412",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "LE = LabelEncoder()\n",
        "train_raw['label'] = LE.fit_transform(train_raw['label'])\n",
        "train_raw.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>In XX/XX/XXXX my wages that I earned at my job...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>XXXX was submitted XX/XX/XXXX. At the time I s...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I spoke to XXXX of green tree representatives ...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>i opened XXXX Bank of America credit cards 15-...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I applied for a loan with XXXX XXXX and had pu...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label\n",
              "0  In XX/XX/XXXX my wages that I earned at my job...      6\n",
              "1  XXXX was submitted XX/XX/XXXX. At the time I s...      6\n",
              "2  I spoke to XXXX of green tree representatives ...      6\n",
              "3  i opened XXXX Bank of America credit cards 15-...      2\n",
              "4  I applied for a loan with XXXX XXXX and had pu...      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvAEbfcK40vP",
        "colab_type": "code",
        "outputId": "1eecd4ae-177d-4578-b3df-adb2f137451f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(np.unique(train_raw['label']))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjTcB2IElYK-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train = train_raw.copy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_KSdpULo4vBM",
        "colab_type": "code",
        "outputId": "fbd5b957-68d9-49fd-dc57-e485fa852c53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train = train.reindex(np.random.permutation(train.index))\n",
        "train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>14660</th>\n",
              "      <td>XXXX XXXX, XXXX Hyundai of XXXX XXXX XXXX XXXX...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15524</th>\n",
              "      <td>On or about XXXX XXXX, XXXX, the XXXX purchase...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9608</th>\n",
              "      <td>On XXXX I became aware that a recently termina...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9602</th>\n",
              "      <td>I recently attempted to obtain a mortgage, at ...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11955</th>\n",
              "      <td>Shortly after purchasing my first home I was i...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "14660  XXXX XXXX, XXXX Hyundai of XXXX XXXX XXXX XXXX...      1\n",
              "15524  On or about XXXX XXXX, XXXX, the XXXX purchase...      6\n",
              "9608   On XXXX I became aware that a recently termina...      0\n",
              "9602   I recently attempted to obtain a mortgage, at ...      4\n",
              "11955  Shortly after purchasing my first home I was i...      6"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CJaaGqEC61Tw",
        "colab_type": "text"
      },
      "source": [
        "Clean the text columns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lFLkBvrnyKnt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "def clean_txt(text):\n",
        "  text = re.sub(\"'\", \"\",text)\n",
        "  text=re.sub(\"(\\\\W)+\",\" \",text)    \n",
        "  return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4yHuGEayROw",
        "colab_type": "code",
        "outputId": "000869c2-f3eb-46c1-8aec-cb54a868d372",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train['text']  = train.text.apply(clean_txt)\n",
        "train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>14660</th>\n",
              "      <td>XXXX XXXX XXXX Hyundai of XXXX XXXX XXXX XXXX ...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15524</th>\n",
              "      <td>On or about XXXX XXXX XXXX the XXXX purchased ...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9608</th>\n",
              "      <td>On XXXX I became aware that a recently termina...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9602</th>\n",
              "      <td>I recently attempted to obtain a mortgage at w...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11955</th>\n",
              "      <td>Shortly after purchasing my first home I was i...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "14660  XXXX XXXX XXXX Hyundai of XXXX XXXX XXXX XXXX ...      1\n",
              "15524  On or about XXXX XXXX XXXX the XXXX purchased ...      6\n",
              "9608   On XXXX I became aware that a recently termina...      0\n",
              "9602   I recently attempted to obtain a mortgage at w...      4\n",
              "11955  Shortly after purchasing my first home I was i...      6"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S3EJew8g5cUK",
        "colab_type": "code",
        "outputId": "5b0fe85f-23df-46ad-f705-776ad8e4d971",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "train, val = train_test_split(train, test_size=0.2, random_state=35)\n",
        "train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6711</th>\n",
              "      <td>I was sold a Timeshare in XXXX XXXX NJ at the ...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13160</th>\n",
              "      <td>I have call and asked this Collection Agency D...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16952</th>\n",
              "      <td>XX XX XXXX my debit card information was stole...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6908</th>\n",
              "      <td>I have serious questions of the ethical behavi...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15676</th>\n",
              "      <td>To whom it may concern I appreciate the email ...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "6711   I was sold a Timeshare in XXXX XXXX NJ at the ...      3\n",
              "13160  I have call and asked this Collection Agency D...      4\n",
              "16952  XX XX XXXX my debit card information was stole...      0\n",
              "6908   I have serious questions of the ethical behavi...      3\n",
              "15676  To whom it may concern I appreciate the email ...      2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Cr7Ch3_5rnH",
        "colab_type": "code",
        "outputId": "288d77ec-56c6-4fb5-b780-4216cf1c9f9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "train.reset_index(drop=True, inplace=True)\n",
        "train.head(2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I was sold a Timeshare in XXXX XXXX NJ at the ...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I have call and asked this Collection Agency D...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label\n",
              "0  I was sold a Timeshare in XXXX XXXX NJ at the ...      3\n",
              "1  I have call and asked this Collection Agency D...      4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6-1O5J9G54hV",
        "colab_type": "code",
        "outputId": "dcc34ce3-2aa4-4c2b-eb27-d23bf4196f6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "val.reset_index(drop=True, inplace=True)\n",
        "val.head(2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>on XXXX XXXX 2015 at XXXX I walked in to Bank ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I have been disputing and requesting validatio...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label\n",
              "0  on XXXX XXXX 2015 at XXXX I walked in to Bank ...      0\n",
              "1  I have been disputing and requesting validatio...      3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ziIsgHqrz0n6",
        "colab_type": "code",
        "outputId": "7020b652-4989-41b1-9e32-c83666c6f9ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "val.shape, train.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((3429, 2), (13713, 2))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TGq2lzLG59SC",
        "colab_type": "code",
        "outputId": "776bf6cb-74ba-40eb-c26a-247b684b574a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "#Installing BERT module\n",
        "!pip install bert-tensorflow"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting bert-tensorflow\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/66/7eb4e8b6ea35b7cc54c322c816f976167a43019750279a8473d355800a93/bert_tensorflow-1.0.1-py2.py3-none-any.whl (67kB)\n",
            "\r\u001b[K     |████▉                           | 10kB 25.2MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 20kB 1.8MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 30kB 2.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 40kB 1.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 51kB 2.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 61kB 2.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 71kB 2.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from bert-tensorflow) (1.12.0)\n",
            "Installing collected packages: bert-tensorflow\n",
            "Successfully installed bert-tensorflow-1.0.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zTcMyKKl6DdA",
        "colab_type": "code",
        "outputId": "fc31a0bb-4c45-4b7c-8dc9-752261c06f51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "#Importing BERT modules\n",
        "import bert\n",
        "from bert import run_classifier\n",
        "from bert import optimization\n",
        "from bert import tokenization"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ej8QOozZbEva",
        "colab_type": "text"
      },
      "source": [
        "# Setting The Output Directory for BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NNPhIMrr6ra9",
        "colab_type": "code",
        "outputId": "ffc63c79-3bea-481d-8263-55f3e65decdb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Set the output directory for saving model file\n",
        "OUTPUT_DIR = '/bert_news_category'\n",
        "\n",
        "#@markdown Whether or not to clear/delete the directory and create a new one\n",
        "DO_DELETE = True #@param {type:\"boolean\"}\n",
        "\n",
        "if DO_DELETE:\n",
        "  try:\n",
        "    tf.gfile.DeleteRecursively(OUTPUT_DIR)\n",
        "  except:\n",
        "    pass\n",
        "\n",
        "tf.gfile.MakeDirs(OUTPUT_DIR)\n",
        "print('***** Model output directory: {} *****'.format(OUTPUT_DIR))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "***** Model output directory: /bert_news_category *****\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0YGONt0p60Ay",
        "colab_type": "code",
        "outputId": "5b986fab-16ed-4613-d2c9-554765ed86bd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(\"Training Set Shape :\", train.shape)\n",
        "print(\"Validation Set Shape :\", val.shape)\n",
        "# print(\"Test Set Shape :\", test.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training Set Shape : (13713, 2)\n",
            "Validation Set Shape : (3429, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s6NYKx4P7N66",
        "colab_type": "code",
        "outputId": "e7d7aa06-98d7-4d6f-d164-93b721c872e2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "DATA_COLUMN = 'text'\n",
        "LABEL_COLUMN = 'label'\n",
        "# The list containing all the classes (train['SECTION'].unique())\n",
        "label_list = [x for x in np.unique(train.label)]\n",
        "label_list"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oOSEd24bbiUq",
        "colab_type": "text"
      },
      "source": [
        "# Splitting the Data into smaller chunks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ausf5AlOkPCH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_split(text1):\n",
        "  l_total = []\n",
        "  l_parcial = []\n",
        "  if len(text1.split())//150 >0:\n",
        "    n = len(text1.split())//150\n",
        "  else: \n",
        "    n = 1\n",
        "  for w in range(n):\n",
        "    if w == 0:\n",
        "      l_parcial = text1.split()[:200]\n",
        "      l_total.append(\" \".join(l_parcial))\n",
        "    else:\n",
        "      l_parcial = text1.split()[w*150:w*150 + 200]\n",
        "      l_total.append(\" \".join(l_parcial))\n",
        "  return l_total"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E-u6mDkbLpTY",
        "colab_type": "code",
        "outputId": "972100f3-569c-4a27-a24c-3fb2728d3581",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train['text_split'] = train[DATA_COLUMN].apply(get_split)\n",
        "train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>text_split</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I was sold a Timeshare in XXXX XXXX NJ at the ...</td>\n",
              "      <td>3</td>\n",
              "      <td>[I was sold a Timeshare in XXXX XXXX NJ at the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I have call and asked this Collection Agency D...</td>\n",
              "      <td>4</td>\n",
              "      <td>[I have call and asked this Collection Agency ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>XX XX XXXX my debit card information was stole...</td>\n",
              "      <td>0</td>\n",
              "      <td>[XX XX XXXX my debit card information was stol...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I have serious questions of the ethical behavi...</td>\n",
              "      <td>3</td>\n",
              "      <td>[I have serious questions of the ethical behav...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>To whom it may concern I appreciate the email ...</td>\n",
              "      <td>2</td>\n",
              "      <td>[To whom it may concern I appreciate the email...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  ...                                         text_split\n",
              "0  I was sold a Timeshare in XXXX XXXX NJ at the ...  ...  [I was sold a Timeshare in XXXX XXXX NJ at the...\n",
              "1  I have call and asked this Collection Agency D...  ...  [I have call and asked this Collection Agency ...\n",
              "2  XX XX XXXX my debit card information was stole...  ...  [XX XX XXXX my debit card information was stol...\n",
              "3  I have serious questions of the ethical behavi...  ...  [I have serious questions of the ethical behav...\n",
              "4  To whom it may concern I appreciate the email ...  ...  [To whom it may concern I appreciate the email...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zrlehCFUplB",
        "colab_type": "code",
        "outputId": "7bd7c47a-adae-418a-c50e-29b2deb3c7d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "val['text_split'] = val[DATA_COLUMN].apply(get_split)\n",
        "val.head(2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "      <th>text_split</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>on XXXX XXXX 2015 at XXXX I walked in to Bank ...</td>\n",
              "      <td>0</td>\n",
              "      <td>[on XXXX XXXX 2015 at XXXX I walked in to Bank...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I have been disputing and requesting validatio...</td>\n",
              "      <td>3</td>\n",
              "      <td>[I have been disputing and requesting validati...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  ...                                         text_split\n",
              "0  on XXXX XXXX 2015 at XXXX I walked in to Bank ...  ...  [on XXXX XXXX 2015 at XXXX I walked in to Bank...\n",
              "1  I have been disputing and requesting validatio...  ...  [I have been disputing and requesting validati...\n",
              "\n",
              "[2 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5_zMerj1VGaM",
        "colab_type": "code",
        "outputId": "cc839f6a-8be6-40a9-aaa9-29752d8d5fff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_l = []\n",
        "label_l = []\n",
        "index_l =[]\n",
        "for idx,row in train.iterrows():\n",
        "  for l in row['text_split']:\n",
        "    train_l.append(l)\n",
        "    label_l.append(row['label'])\n",
        "    index_l.append(idx)\n",
        "len(train_l), len(label_l), len(index_l)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(31679, 31679, 31679)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rBrXEaxHVNG4",
        "colab_type": "code",
        "outputId": "de4dbc34-1fde-4233-a902-ab522cfc9520",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "val_l = []\n",
        "val_label_l = []\n",
        "val_index_l = []\n",
        "for idx,row in val.iterrows():\n",
        "  for l in row['text_split']:\n",
        "    val_l.append(l)\n",
        "    val_label_l.append(row['label'])\n",
        "    val_index_l.append(idx)\n",
        "len(val_l), len(val_label_l), len(val_index_l)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(7946, 7946, 7946)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hu5Sx8Rm0bAj",
        "colab_type": "text"
      },
      "source": [
        "The final dataset for training:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mojRk8kWVVB4",
        "colab_type": "code",
        "outputId": "95c4ea0f-7800-47b0-fa8a-9005671da2a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train_df = pd.DataFrame({DATA_COLUMN:train_l, LABEL_COLUMN:label_l})\n",
        "train_df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I was sold a Timeshare in XXXX XXXX NJ at the ...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>unless they could prove to me any sort of valu...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I have call and asked this Collection Agency D...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>XX XX XXXX my debit card information was stole...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I have serious questions of the ethical behavi...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label\n",
              "0  I was sold a Timeshare in XXXX XXXX NJ at the ...      3\n",
              "1  unless they could prove to me any sort of valu...      3\n",
              "2  I have call and asked this Collection Agency D...      4\n",
              "3  XX XX XXXX my debit card information was stole...      0\n",
              "4  I have serious questions of the ethical behavi...      3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k_I-ZbSKVmrZ",
        "colab_type": "code",
        "outputId": "ab55b3b8-5524-4cb3-bb23-a0a10c3a2bf0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "val_df = pd.DataFrame({DATA_COLUMN:val_l, LABEL_COLUMN:val_label_l})\n",
        "val_df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>on XXXX XXXX 2015 at XXXX I walked in to Bank ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>for her to train because I know they do nt nee...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I have been disputing and requesting validatio...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>the right to request validation of the debt yo...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>My complaint is against the credit bureau Equi...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text  label\n",
              "0  on XXXX XXXX 2015 at XXXX I walked in to Bank ...      0\n",
              "1  for her to train because I know they do nt nee...      0\n",
              "2  I have been disputing and requesting validatio...      3\n",
              "3  the right to request validation of the debt yo...      3\n",
              "4  My complaint is against the credit bureau Equi...      3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "og08g7DScPtK",
        "colab_type": "text"
      },
      "source": [
        "# BERT: Data Preprocessing "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2PWVomvm7TV5",
        "colab_type": "text"
      },
      "source": [
        "Process the data for BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e-_zLSnh7evE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_InputExamples = train_df.apply(lambda x: bert.run_classifier.InputExample(guid=None,\n",
        "                                                                   text_a = x[DATA_COLUMN], \n",
        "                                                                   text_b = None, \n",
        "                                                                   label = x[LABEL_COLUMN]), axis = 1)\n",
        "\n",
        "val_InputExamples = val_df.apply(lambda x: bert.run_classifier.InputExample(guid=None, \n",
        "                                                                   text_a = x[DATA_COLUMN], \n",
        "                                                                   text_b = None, \n",
        "                                                                   label = x[LABEL_COLUMN]), axis = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UOq7ETNe7zGJ",
        "colab_type": "code",
        "outputId": "0c86519a-5376-4b21-be65-9c12f99afb6f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "train_InputExamples"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0        <bert.run_classifier.InputExample object at 0x...\n",
              "1        <bert.run_classifier.InputExample object at 0x...\n",
              "2        <bert.run_classifier.InputExample object at 0x...\n",
              "3        <bert.run_classifier.InputExample object at 0x...\n",
              "4        <bert.run_classifier.InputExample object at 0x...\n",
              "                               ...                        \n",
              "31674    <bert.run_classifier.InputExample object at 0x...\n",
              "31675    <bert.run_classifier.InputExample object at 0x...\n",
              "31676    <bert.run_classifier.InputExample object at 0x...\n",
              "31677    <bert.run_classifier.InputExample object at 0x...\n",
              "31678    <bert.run_classifier.InputExample object at 0x...\n",
              "Length: 31679, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d0E8U0OQ8VW6",
        "colab_type": "code",
        "outputId": "0934f3f9-f7ae-44dc-fbef-4e038c8bb99a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "print(\"Row 0 - guid of training set : \", train_InputExamples.iloc[0].guid)\n",
        "print(\"\\n__________\\nRow 0 - text_a of training set : \", train_InputExamples.iloc[0].text_a)\n",
        "print(\"\\n__________\\nRow 0 - text_b of training set : \", train_InputExamples.iloc[0].text_b)\n",
        "print(\"\\n__________\\nRow 0 - label of training set : \", train_InputExamples.iloc[0].label)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Row 0 - guid of training set :  None\n",
            "\n",
            "__________\n",
            "Row 0 - text_a of training set :  I was sold a Timeshare in XXXX XXXX NJ at the XXXX XXXX through a company doing business as XXXX Subsequent to the agreement I had an issue where my wife took ill and needed surgery Since she is the primary income provider I found myself unable to meet all of my obligations I determined the Timeshare was an unneeded expense so they were not paid They started harassing me I looked into this whole timeshare issue and found out that my membership was nothing more than a source of income for the Hotel that if I were to simply reserve a room through the Internet at the exact same location it would cost me less than HALF what I pay annually in Maintenance fees and taxes When they called back after I discovered this I informed them that I was misled and refused to pay any more into this unless they could prove to me any sort of value in what I purchased They have since then threatened me with legal action including Check Fraud and when I informed them that their threats were in fact a violation of the Fair Debt Collection Act they stopped the calls but\n",
            "\n",
            "__________\n",
            "Row 0 - text_b of training set :  None\n",
            "\n",
            "__________\n",
            "Row 0 - label of training set :  3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SoIt5AHadACM",
        "colab_type": "text"
      },
      "source": [
        "# BERT: Loading the pre-trained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W4PZ8ogj8ae2",
        "colab_type": "code",
        "outputId": "26a10fed-e770-41a2-e716-11d5cc7dc2ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "\n",
        "# This is a path to an uncased (all lowercase) version of BERT\n",
        "BERT_MODEL_HUB = \"https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1\"\n",
        "\n",
        "def create_tokenizer_from_hub_module():\n",
        "  \"\"\"Get the vocab file and casing info from the Hub module.\"\"\"\n",
        "  with tf.Graph().as_default():\n",
        "    bert_module = hub.Module(BERT_MODEL_HUB)\n",
        "    tokenization_info = bert_module(signature=\"tokenization_info\", as_dict=True)\n",
        "    with tf.Session() as sess:\n",
        "      vocab_file, do_lower_case = sess.run([tokenization_info[\"vocab_file\"],\n",
        "                                            tokenization_info[\"do_lower_case\"]])\n",
        "      \n",
        "  return bert.tokenization.FullTokenizer(\n",
        "      vocab_file=vocab_file, do_lower_case=do_lower_case)\n",
        "\n",
        "tokenizer = create_tokenizer_from_hub_module()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/tokenization.py:125: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/tokenization.py:125: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qS_ybJmv8lye",
        "colab_type": "code",
        "outputId": "2c48b73b-eacc-4073-defc-934eaf83a24f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(tokenizer.vocab.keys())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "30522"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O6OqoZjv8r27",
        "colab_type": "code",
        "outputId": "bad53d19-4e97-47cb-ad7c-3781ebb5e303",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "#Here is what the tokenised sample of the first training set observation looks like\n",
        "print(tokenizer.tokenize(train_InputExamples.iloc[0].text_a))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['i', 'was', 'sold', 'a', 'times', '##har', '##e', 'in', 'xx', '##xx', 'xx', '##xx', 'nj', 'at', 'the', 'xx', '##xx', 'xx', '##xx', 'through', 'a', 'company', 'doing', 'business', 'as', 'xx', '##xx', 'subsequent', 'to', 'the', 'agreement', 'i', 'had', 'an', 'issue', 'where', 'my', 'wife', 'took', 'ill', 'and', 'needed', 'surgery', 'since', 'she', 'is', 'the', 'primary', 'income', 'provider', 'i', 'found', 'myself', 'unable', 'to', 'meet', 'all', 'of', 'my', 'obligations', 'i', 'determined', 'the', 'times', '##har', '##e', 'was', 'an', 'un', '##nee', '##ded', 'expense', 'so', 'they', 'were', 'not', 'paid', 'they', 'started', 'hara', '##ssing', 'me', 'i', 'looked', 'into', 'this', 'whole', 'times', '##har', '##e', 'issue', 'and', 'found', 'out', 'that', 'my', 'membership', 'was', 'nothing', 'more', 'than', 'a', 'source', 'of', 'income', 'for', 'the', 'hotel', 'that', 'if', 'i', 'were', 'to', 'simply', 'reserve', 'a', 'room', 'through', 'the', 'internet', 'at', 'the', 'exact', 'same', 'location', 'it', 'would', 'cost', 'me', 'less', 'than', 'half', 'what', 'i', 'pay', 'annually', 'in', 'maintenance', 'fees', 'and', 'taxes', 'when', 'they', 'called', 'back', 'after', 'i', 'discovered', 'this', 'i', 'informed', 'them', 'that', 'i', 'was', 'mis', '##led', 'and', 'refused', 'to', 'pay', 'any', 'more', 'into', 'this', 'unless', 'they', 'could', 'prove', 'to', 'me', 'any', 'sort', 'of', 'value', 'in', 'what', 'i', 'purchased', 'they', 'have', 'since', 'then', 'threatened', 'me', 'with', 'legal', 'action', 'including', 'check', 'fraud', 'and', 'when', 'i', 'informed', 'them', 'that', 'their', 'threats', 'were', 'in', 'fact', 'a', 'violation', 'of', 'the', 'fair', 'debt', 'collection', 'act', 'they', 'stopped', 'the', 'calls', 'but']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q87k_orF8vpz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_SEQ_LENGTH = 200"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G_LBy-yy-GSU",
        "colab_type": "code",
        "outputId": "a1c02dff-0cd3-4ef8-c013-ced8dd6b484f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Convert our train and validation features to InputFeatures that BERT understands.\n",
        "train_features = bert.run_classifier.convert_examples_to_features(train_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)\n",
        "\n",
        "val_features = bert.run_classifier.convert_examples_to_features(val_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/run_classifier.py:774: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/run_classifier.py:774: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 0 of 31679\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 0 of 31679\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i was sold a times ##har ##e in xx ##xx xx ##xx nj at the xx ##xx xx ##xx through a company doing business as xx ##xx subsequent to the agreement i had an issue where my wife took ill and needed surgery since she is the primary income provider i found myself unable to meet all of my obligations i determined the times ##har ##e was an un ##nee ##ded expense so they were not paid they started hara ##ssing me i looked into this whole times ##har ##e issue and found out that my membership was nothing more than a source of income for the hotel that if i were to simply reserve a room through the internet at the exact same location it would cost me less than half what i pay annually in maintenance fees and taxes when they called back after i discovered this i informed them that i was mis ##led and refused to pay any more into this unless they could prove to me any sort of value in what i purchased they have since then threatened me with legal action including check fraud and when i informed them that their [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i was sold a times ##har ##e in xx ##xx xx ##xx nj at the xx ##xx xx ##xx through a company doing business as xx ##xx subsequent to the agreement i had an issue where my wife took ill and needed surgery since she is the primary income provider i found myself unable to meet all of my obligations i determined the times ##har ##e was an un ##nee ##ded expense so they were not paid they started hara ##ssing me i looked into this whole times ##har ##e issue and found out that my membership was nothing more than a source of income for the hotel that if i were to simply reserve a room through the internet at the exact same location it would cost me less than half what i pay annually in maintenance fees and taxes when they called back after i discovered this i informed them that i was mis ##led and refused to pay any more into this unless they could prove to me any sort of value in what i purchased they have since then threatened me with legal action including check fraud and when i informed them that their [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2001 2853 1037 2335 8167 2063 1999 22038 20348 22038 20348 19193 2012 1996 22038 20348 22038 20348 2083 1037 2194 2725 2449 2004 22038 20348 4745 2000 1996 3820 1045 2018 2019 3277 2073 2026 2564 2165 5665 1998 2734 5970 2144 2016 2003 1996 3078 3318 10802 1045 2179 2870 4039 2000 3113 2035 1997 2026 14422 1045 4340 1996 2335 8167 2063 2001 2019 4895 24045 5732 10961 2061 2027 2020 2025 3825 2027 2318 18820 18965 2033 1045 2246 2046 2023 2878 2335 8167 2063 3277 1998 2179 2041 2008 2026 5779 2001 2498 2062 2084 1037 3120 1997 3318 2005 1996 3309 2008 2065 1045 2020 2000 3432 3914 1037 2282 2083 1996 4274 2012 1996 6635 2168 3295 2009 2052 3465 2033 2625 2084 2431 2054 1045 3477 6604 1999 6032 9883 1998 7773 2043 2027 2170 2067 2044 1045 3603 2023 1045 6727 2068 2008 1045 2001 28616 3709 1998 4188 2000 3477 2151 2062 2046 2023 4983 2027 2071 6011 2000 2033 2151 4066 1997 3643 1999 2054 1045 4156 2027 2031 2144 2059 5561 2033 2007 3423 2895 2164 4638 9861 1998 2043 1045 6727 2068 2008 2037 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2001 2853 1037 2335 8167 2063 1999 22038 20348 22038 20348 19193 2012 1996 22038 20348 22038 20348 2083 1037 2194 2725 2449 2004 22038 20348 4745 2000 1996 3820 1045 2018 2019 3277 2073 2026 2564 2165 5665 1998 2734 5970 2144 2016 2003 1996 3078 3318 10802 1045 2179 2870 4039 2000 3113 2035 1997 2026 14422 1045 4340 1996 2335 8167 2063 2001 2019 4895 24045 5732 10961 2061 2027 2020 2025 3825 2027 2318 18820 18965 2033 1045 2246 2046 2023 2878 2335 8167 2063 3277 1998 2179 2041 2008 2026 5779 2001 2498 2062 2084 1037 3120 1997 3318 2005 1996 3309 2008 2065 1045 2020 2000 3432 3914 1037 2282 2083 1996 4274 2012 1996 6635 2168 3295 2009 2052 3465 2033 2625 2084 2431 2054 1045 3477 6604 1999 6032 9883 1998 7773 2043 2027 2170 2067 2044 1045 3603 2023 1045 6727 2068 2008 1045 2001 28616 3709 1998 4188 2000 3477 2151 2062 2046 2023 4983 2027 2071 6011 2000 2033 2151 4066 1997 3643 1999 2054 1045 4156 2027 2031 2144 2059 5561 2033 2007 3423 2895 2164 4638 9861 1998 2043 1045 6727 2068 2008 2037 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] unless they could prove to me any sort of value in what i purchased they have since then threatened me with legal action including check fraud and when i informed them that their threats were in fact a violation of the fair debt collection act they stopped the calls but now have started doing hard inquiries into my credit xx ##xx on xx ##xx xx ##xx 2015 and the latest on xx ##xx xx ##xx 2015 i called trans ##uni ##on and they informed me that the only way to get these unauthorized inquiries removed was to contact xx ##xx in writing and ask them to allow the inquiries to be removed i find this totally unfair and as i understand illegal as i have not authorized these multiple inquiries into my credit and as a result have suffered a xx ##xx point drop in my credit score i ask for your assistance in getting this unfair practice stopped and that xx ##xx be ordered to cease and des ##ist from these practices this is nothing more than a sc ##am in my opinion and their utilization of these tactics to damage my credit score nothing more than [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] unless they could prove to me any sort of value in what i purchased they have since then threatened me with legal action including check fraud and when i informed them that their threats were in fact a violation of the fair debt collection act they stopped the calls but now have started doing hard inquiries into my credit xx ##xx on xx ##xx xx ##xx 2015 and the latest on xx ##xx xx ##xx 2015 i called trans ##uni ##on and they informed me that the only way to get these unauthorized inquiries removed was to contact xx ##xx in writing and ask them to allow the inquiries to be removed i find this totally unfair and as i understand illegal as i have not authorized these multiple inquiries into my credit and as a result have suffered a xx ##xx point drop in my credit score i ask for your assistance in getting this unfair practice stopped and that xx ##xx be ordered to cease and des ##ist from these practices this is nothing more than a sc ##am in my opinion and their utilization of these tactics to damage my credit score nothing more than [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 4983 2027 2071 6011 2000 2033 2151 4066 1997 3643 1999 2054 1045 4156 2027 2031 2144 2059 5561 2033 2007 3423 2895 2164 4638 9861 1998 2043 1045 6727 2068 2008 2037 8767 2020 1999 2755 1037 11371 1997 1996 4189 7016 3074 2552 2027 3030 1996 4455 2021 2085 2031 2318 2725 2524 27050 2046 2026 4923 22038 20348 2006 22038 20348 22038 20348 2325 1998 1996 6745 2006 22038 20348 22038 20348 2325 1045 2170 9099 19496 2239 1998 2027 6727 2033 2008 1996 2069 2126 2000 2131 2122 24641 27050 3718 2001 2000 3967 22038 20348 1999 3015 1998 3198 2068 2000 3499 1996 27050 2000 2022 3718 1045 2424 2023 6135 15571 1998 2004 1045 3305 6206 2004 1045 2031 2025 9362 2122 3674 27050 2046 2026 4923 1998 2004 1037 2765 2031 4265 1037 22038 20348 2391 4530 1999 2026 4923 3556 1045 3198 2005 2115 5375 1999 2893 2023 15571 3218 3030 1998 2008 22038 20348 2022 3641 2000 13236 1998 4078 2923 2013 2122 6078 2023 2003 2498 2062 2084 1037 8040 3286 1999 2026 5448 1998 2037 27891 1997 2122 9887 2000 4053 2026 4923 3556 2498 2062 2084 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 4983 2027 2071 6011 2000 2033 2151 4066 1997 3643 1999 2054 1045 4156 2027 2031 2144 2059 5561 2033 2007 3423 2895 2164 4638 9861 1998 2043 1045 6727 2068 2008 2037 8767 2020 1999 2755 1037 11371 1997 1996 4189 7016 3074 2552 2027 3030 1996 4455 2021 2085 2031 2318 2725 2524 27050 2046 2026 4923 22038 20348 2006 22038 20348 22038 20348 2325 1998 1996 6745 2006 22038 20348 22038 20348 2325 1045 2170 9099 19496 2239 1998 2027 6727 2033 2008 1996 2069 2126 2000 2131 2122 24641 27050 3718 2001 2000 3967 22038 20348 1999 3015 1998 3198 2068 2000 3499 1996 27050 2000 2022 3718 1045 2424 2023 6135 15571 1998 2004 1045 3305 6206 2004 1045 2031 2025 9362 2122 3674 27050 2046 2026 4923 1998 2004 1037 2765 2031 4265 1037 22038 20348 2391 4530 1999 2026 4923 3556 1045 3198 2005 2115 5375 1999 2893 2023 15571 3218 3030 1998 2008 22038 20348 2022 3641 2000 13236 1998 4078 2923 2013 2122 6078 2023 2003 2498 2062 2084 1037 8040 3286 1999 2026 5448 1998 2037 27891 1997 2122 9887 2000 4053 2026 4923 3556 2498 2062 2084 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i have call and asked this collection agency diversified consultant to remove a collection of 170 00 from xx ##xx xx ##xx xx ##xx they claim they have the authority to collect the first time i called they could nt find that i owed a debt and the second time i called they told me that they had an account but my social security number did nt match the xx ##xx they had on the account this is not my debt i have never had any type of service with xx ##xx xx ##xx xx ##xx i asked for a verification of the debt and ended up talking with a supervisor named xx ##xx xx ##xx direct phone xx ##xx xx ##xx was very helpful and informed me that the social security associated with this collection account was different than mine he said they would remove del ##ete this collection from all xx ##xx of the credit reporting agencies xx ##xx xx ##xx and xx ##xx xx ##xx if i would fa ##x them some information he instructed me to fa ##x front page of my credit report and the page showing the collection account in addition to [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i have call and asked this collection agency diversified consultant to remove a collection of 170 00 from xx ##xx xx ##xx xx ##xx they claim they have the authority to collect the first time i called they could nt find that i owed a debt and the second time i called they told me that they had an account but my social security number did nt match the xx ##xx they had on the account this is not my debt i have never had any type of service with xx ##xx xx ##xx xx ##xx i asked for a verification of the debt and ended up talking with a supervisor named xx ##xx xx ##xx direct phone xx ##xx xx ##xx was very helpful and informed me that the social security associated with this collection account was different than mine he said they would remove del ##ete this collection from all xx ##xx of the credit reporting agencies xx ##xx xx ##xx and xx ##xx xx ##xx if i would fa ##x them some information he instructed me to fa ##x front page of my credit report and the page showing the collection account in addition to [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2031 2655 1998 2356 2023 3074 4034 24908 8930 2000 6366 1037 3074 1997 10894 4002 2013 22038 20348 22038 20348 22038 20348 2027 4366 2027 2031 1996 3691 2000 8145 1996 2034 2051 1045 2170 2027 2071 23961 2424 2008 1045 12232 1037 7016 1998 1996 2117 2051 1045 2170 2027 2409 2033 2008 2027 2018 2019 4070 2021 2026 2591 3036 2193 2106 23961 2674 1996 22038 20348 2027 2018 2006 1996 4070 2023 2003 2025 2026 7016 1045 2031 2196 2018 2151 2828 1997 2326 2007 22038 20348 22038 20348 22038 20348 1045 2356 2005 1037 22616 1997 1996 7016 1998 3092 2039 3331 2007 1037 12366 2315 22038 20348 22038 20348 3622 3042 22038 20348 22038 20348 2001 2200 14044 1998 6727 2033 2008 1996 2591 3036 3378 2007 2023 3074 4070 2001 2367 2084 3067 2002 2056 2027 2052 6366 3972 12870 2023 3074 2013 2035 22038 20348 1997 1996 4923 7316 6736 22038 20348 22038 20348 1998 22038 20348 22038 20348 2065 1045 2052 6904 2595 2068 2070 2592 2002 10290 2033 2000 6904 2595 2392 3931 1997 2026 4923 3189 1998 1996 3931 4760 1996 3074 4070 1999 2804 2000 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2031 2655 1998 2356 2023 3074 4034 24908 8930 2000 6366 1037 3074 1997 10894 4002 2013 22038 20348 22038 20348 22038 20348 2027 4366 2027 2031 1996 3691 2000 8145 1996 2034 2051 1045 2170 2027 2071 23961 2424 2008 1045 12232 1037 7016 1998 1996 2117 2051 1045 2170 2027 2409 2033 2008 2027 2018 2019 4070 2021 2026 2591 3036 2193 2106 23961 2674 1996 22038 20348 2027 2018 2006 1996 4070 2023 2003 2025 2026 7016 1045 2031 2196 2018 2151 2828 1997 2326 2007 22038 20348 22038 20348 22038 20348 1045 2356 2005 1037 22616 1997 1996 7016 1998 3092 2039 3331 2007 1037 12366 2315 22038 20348 22038 20348 3622 3042 22038 20348 22038 20348 2001 2200 14044 1998 6727 2033 2008 1996 2591 3036 3378 2007 2023 3074 4070 2001 2367 2084 3067 2002 2056 2027 2052 6366 3972 12870 2023 3074 2013 2035 22038 20348 1997 1996 4923 7316 6736 22038 20348 22038 20348 1998 22038 20348 22038 20348 2065 1045 2052 6904 2595 2068 2070 2592 2002 10290 2033 2000 6904 2595 2392 3931 1997 2026 4923 3189 1998 1996 3931 4760 1996 3074 4070 1999 2804 2000 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 4 (id = 4)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 4 (id = 4)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] xx xx xx ##xx my de ##bit card information was stolen and someone rack ##ed up about 2000 00 worth of charges on my ci ##ti ##bank checking account when i went into the bank and explained my situation to the banker they told me they would investigate and my money would be ref ##und ##ed within 7 to 10 days i went back to the bank weeks later and still no ref ##und i spoke with a representative from ci ##ti ##bank and she told me they sent me a letter in the mail that i never received that stated they were denying my claim and were nt going to ref ##und my money to my account because it appears i rack ##ed up charges myself she told me i needed to file a police report and there s nothing she can do so i filed my police report like she told me with an officer named xx ##xx xx ##xx in his report he states that he received sur ##va ##ila ##nce from one of the atm s i had claimed to never been to before and found that somebody else was in fact at the [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] xx xx xx ##xx my de ##bit card information was stolen and someone rack ##ed up about 2000 00 worth of charges on my ci ##ti ##bank checking account when i went into the bank and explained my situation to the banker they told me they would investigate and my money would be ref ##und ##ed within 7 to 10 days i went back to the bank weeks later and still no ref ##und i spoke with a representative from ci ##ti ##bank and she told me they sent me a letter in the mail that i never received that stated they were denying my claim and were nt going to ref ##und my money to my account because it appears i rack ##ed up charges myself she told me i needed to file a police report and there s nothing she can do so i filed my police report like she told me with an officer named xx ##xx xx ##xx in his report he states that he received sur ##va ##ila ##nce from one of the atm s i had claimed to never been to before and found that somebody else was in fact at the [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 22038 22038 22038 20348 2026 2139 16313 4003 2592 2001 7376 1998 2619 14513 2098 2039 2055 2456 4002 4276 1997 5571 2006 2026 25022 3775 9299 9361 4070 2043 1045 2253 2046 1996 2924 1998 4541 2026 3663 2000 1996 13448 2027 2409 2033 2027 2052 8556 1998 2026 2769 2052 2022 25416 8630 2098 2306 1021 2000 2184 2420 1045 2253 2067 2000 1996 2924 3134 2101 1998 2145 2053 25416 8630 1045 3764 2007 1037 4387 2013 25022 3775 9299 1998 2016 2409 2033 2027 2741 2033 1037 3661 1999 1996 5653 2008 1045 2196 2363 2008 3090 2027 2020 16039 2026 4366 1998 2020 23961 2183 2000 25416 8630 2026 2769 2000 2026 4070 2138 2009 3544 1045 14513 2098 2039 5571 2870 2016 2409 2033 1045 2734 2000 5371 1037 2610 3189 1998 2045 1055 2498 2016 2064 2079 2061 1045 6406 2026 2610 3189 2066 2016 2409 2033 2007 2019 2961 2315 22038 20348 22038 20348 1999 2010 3189 2002 2163 2008 2002 2363 7505 3567 11733 5897 2013 2028 1997 1996 27218 1055 1045 2018 3555 2000 2196 2042 2000 2077 1998 2179 2008 8307 2842 2001 1999 2755 2012 1996 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 22038 22038 22038 20348 2026 2139 16313 4003 2592 2001 7376 1998 2619 14513 2098 2039 2055 2456 4002 4276 1997 5571 2006 2026 25022 3775 9299 9361 4070 2043 1045 2253 2046 1996 2924 1998 4541 2026 3663 2000 1996 13448 2027 2409 2033 2027 2052 8556 1998 2026 2769 2052 2022 25416 8630 2098 2306 1021 2000 2184 2420 1045 2253 2067 2000 1996 2924 3134 2101 1998 2145 2053 25416 8630 1045 3764 2007 1037 4387 2013 25022 3775 9299 1998 2016 2409 2033 2027 2741 2033 1037 3661 1999 1996 5653 2008 1045 2196 2363 2008 3090 2027 2020 16039 2026 4366 1998 2020 23961 2183 2000 25416 8630 2026 2769 2000 2026 4070 2138 2009 3544 1045 14513 2098 2039 5571 2870 2016 2409 2033 1045 2734 2000 5371 1037 2610 3189 1998 2045 1055 2498 2016 2064 2079 2061 1045 6406 2026 2610 3189 2066 2016 2409 2033 2007 2019 2961 2315 22038 20348 22038 20348 1999 2010 3189 2002 2163 2008 2002 2363 7505 3567 11733 5897 2013 2028 1997 1996 27218 1055 1045 2018 3555 2000 2196 2042 2000 2077 1998 2179 2008 8307 2842 2001 1999 2755 2012 1996 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 0 (id = 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 0 (id = 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i have serious questions of the ethical behaviour of ex ##per ##ian in relation to what credit score they report they report on an individual depend ##ant upon whether the individual requesting the credit report is paying for the premium xx ##xx agency report or whether they choose to use a free service provided through their bank or other financial institution xx ##xx in this case where they en ##tic ##e you to enroll in the other xx ##xx agency premium credit report and protection i am going to provide xx ##xx separate ex ##per ##ian credit reports that were pulled within minutes of each other both pu ##rp ##ort ##ing to show my credit score however instead of matching as xx ##xx would expect them to the paid ex ##per ##ian xx ##xx agency report that was signed up for through xx ##xx xx ##xx xx ##xx shows my score to be xx ##xx yet when the free credit score monitoring service provided by ex ##per ##ian for xx ##xx is pulled within minutes and the credit report shows no material differences it reports that my credit score is xx ##xx a full xx ##xx points lower [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i have serious questions of the ethical behaviour of ex ##per ##ian in relation to what credit score they report they report on an individual depend ##ant upon whether the individual requesting the credit report is paying for the premium xx ##xx agency report or whether they choose to use a free service provided through their bank or other financial institution xx ##xx in this case where they en ##tic ##e you to enroll in the other xx ##xx agency premium credit report and protection i am going to provide xx ##xx separate ex ##per ##ian credit reports that were pulled within minutes of each other both pu ##rp ##ort ##ing to show my credit score however instead of matching as xx ##xx would expect them to the paid ex ##per ##ian xx ##xx agency report that was signed up for through xx ##xx xx ##xx xx ##xx shows my score to be xx ##xx yet when the free credit score monitoring service provided by ex ##per ##ian for xx ##xx is pulled within minutes and the credit report shows no material differences it reports that my credit score is xx ##xx a full xx ##xx points lower [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2031 3809 3980 1997 1996 12962 9164 1997 4654 4842 2937 1999 7189 2000 2054 4923 3556 2027 3189 2027 3189 2006 2019 3265 12530 4630 2588 3251 1996 3265 17942 1996 4923 3189 2003 7079 2005 1996 12882 22038 20348 4034 3189 2030 3251 2027 5454 2000 2224 1037 2489 2326 3024 2083 2037 2924 2030 2060 3361 5145 22038 20348 1999 2023 2553 2073 2027 4372 4588 2063 2017 2000 25612 1999 1996 2060 22038 20348 4034 12882 4923 3189 1998 3860 1045 2572 2183 2000 3073 22038 20348 3584 4654 4842 2937 4923 4311 2008 2020 2766 2306 2781 1997 2169 2060 2119 16405 14536 11589 2075 2000 2265 2026 4923 3556 2174 2612 1997 9844 2004 22038 20348 2052 5987 2068 2000 1996 3825 4654 4842 2937 22038 20348 4034 3189 2008 2001 2772 2039 2005 2083 22038 20348 22038 20348 22038 20348 3065 2026 3556 2000 2022 22038 20348 2664 2043 1996 2489 4923 3556 8822 2326 3024 2011 4654 4842 2937 2005 22038 20348 2003 2766 2306 2781 1998 1996 4923 3189 3065 2053 3430 5966 2009 4311 2008 2026 4923 3556 2003 22038 20348 1037 2440 22038 20348 2685 2896 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2031 3809 3980 1997 1996 12962 9164 1997 4654 4842 2937 1999 7189 2000 2054 4923 3556 2027 3189 2027 3189 2006 2019 3265 12530 4630 2588 3251 1996 3265 17942 1996 4923 3189 2003 7079 2005 1996 12882 22038 20348 4034 3189 2030 3251 2027 5454 2000 2224 1037 2489 2326 3024 2083 2037 2924 2030 2060 3361 5145 22038 20348 1999 2023 2553 2073 2027 4372 4588 2063 2017 2000 25612 1999 1996 2060 22038 20348 4034 12882 4923 3189 1998 3860 1045 2572 2183 2000 3073 22038 20348 3584 4654 4842 2937 4923 4311 2008 2020 2766 2306 2781 1997 2169 2060 2119 16405 14536 11589 2075 2000 2265 2026 4923 3556 2174 2612 1997 9844 2004 22038 20348 2052 5987 2068 2000 1996 3825 4654 4842 2937 22038 20348 4034 3189 2008 2001 2772 2039 2005 2083 22038 20348 22038 20348 22038 20348 3065 2026 3556 2000 2022 22038 20348 2664 2043 1996 2489 4923 3556 8822 2326 3024 2011 4654 4842 2937 2005 22038 20348 2003 2766 2306 2781 1998 1996 4923 3189 3065 2053 3430 5966 2009 4311 2008 2026 4923 3556 2003 22038 20348 1037 2440 22038 20348 2685 2896 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 10000 of 31679\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 10000 of 31679\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 20000 of 31679\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 20000 of 31679\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 30000 of 31679\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 30000 of 31679\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 0 of 7946\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Writing example 0 of 7946\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] on xx ##xx xx ##xx 2015 at xx ##xx i walked in to bank of america in xx ##xx xx ##xx xx ##xx xx ##xx tn xx ##xx to deposit cash more than 10 00 that in business require to fill up ct ##r form that i have already did since i ve been customer for many years everything went well until up to that form pop out on the cash ##iers screen it took her 10 minutes trying to figure out the way out then finally called manager that does nt know how to do that either then called another senior cash ##ier cash ##iers manager to help out they chat for a while then she left but this cash ##ier still seems did nt know and still try and try 20 minutes has passed i told her if i can get my dl back she can make a copy of it if she needs to but do nt hold me there for her to train because i know they do nt need me there at all but she refused to give my dl back or make any note just keep trying few more minutes passed i [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] on xx ##xx xx ##xx 2015 at xx ##xx i walked in to bank of america in xx ##xx xx ##xx xx ##xx xx ##xx tn xx ##xx to deposit cash more than 10 00 that in business require to fill up ct ##r form that i have already did since i ve been customer for many years everything went well until up to that form pop out on the cash ##iers screen it took her 10 minutes trying to figure out the way out then finally called manager that does nt know how to do that either then called another senior cash ##ier cash ##iers manager to help out they chat for a while then she left but this cash ##ier still seems did nt know and still try and try 20 minutes has passed i told her if i can get my dl back she can make a copy of it if she needs to but do nt hold me there for her to train because i know they do nt need me there at all but she refused to give my dl back or make any note just keep trying few more minutes passed i [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 2006 22038 20348 22038 20348 2325 2012 22038 20348 1045 2939 1999 2000 2924 1997 2637 1999 22038 20348 22038 20348 22038 20348 22038 20348 28286 22038 20348 2000 12816 5356 2062 2084 2184 4002 2008 1999 2449 5478 2000 6039 2039 14931 2099 2433 2008 1045 2031 2525 2106 2144 1045 2310 2042 8013 2005 2116 2086 2673 2253 2092 2127 2039 2000 2008 2433 3769 2041 2006 1996 5356 10136 3898 2009 2165 2014 2184 2781 2667 2000 3275 2041 1996 2126 2041 2059 2633 2170 3208 2008 2515 23961 2113 2129 2000 2079 2008 2593 2059 2170 2178 3026 5356 3771 5356 10136 3208 2000 2393 2041 2027 11834 2005 1037 2096 2059 2016 2187 2021 2023 5356 3771 2145 3849 2106 23961 2113 1998 2145 3046 1998 3046 2322 2781 2038 2979 1045 2409 2014 2065 1045 2064 2131 2026 21469 2067 2016 2064 2191 1037 6100 1997 2009 2065 2016 3791 2000 2021 2079 23961 2907 2033 2045 2005 2014 2000 3345 2138 1045 2113 2027 2079 23961 2342 2033 2045 2012 2035 2021 2016 4188 2000 2507 2026 21469 2067 2030 2191 2151 3602 2074 2562 2667 2261 2062 2781 2979 1045 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 2006 22038 20348 22038 20348 2325 2012 22038 20348 1045 2939 1999 2000 2924 1997 2637 1999 22038 20348 22038 20348 22038 20348 22038 20348 28286 22038 20348 2000 12816 5356 2062 2084 2184 4002 2008 1999 2449 5478 2000 6039 2039 14931 2099 2433 2008 1045 2031 2525 2106 2144 1045 2310 2042 8013 2005 2116 2086 2673 2253 2092 2127 2039 2000 2008 2433 3769 2041 2006 1996 5356 10136 3898 2009 2165 2014 2184 2781 2667 2000 3275 2041 1996 2126 2041 2059 2633 2170 3208 2008 2515 23961 2113 2129 2000 2079 2008 2593 2059 2170 2178 3026 5356 3771 5356 10136 3208 2000 2393 2041 2027 11834 2005 1037 2096 2059 2016 2187 2021 2023 5356 3771 2145 3849 2106 23961 2113 1998 2145 3046 1998 3046 2322 2781 2038 2979 1045 2409 2014 2065 1045 2064 2131 2026 21469 2067 2016 2064 2191 1037 6100 1997 2009 2065 2016 3791 2000 2021 2079 23961 2907 2033 2045 2005 2014 2000 3345 2138 1045 2113 2027 2079 23961 2342 2033 2045 2012 2035 2021 2016 4188 2000 2507 2026 21469 2067 2030 2191 2151 3602 2074 2562 2667 2261 2062 2781 2979 1045 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 0 (id = 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 0 (id = 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] for her to train because i know they do nt need me there at all but she refused to give my dl back or make any note just keep trying few more minutes passed i asked one more time and she refused then i tried to grab my dl but she took it faster and step back i was furious then i grab her hand try to pull her so i can get my dl but she still refuse to give it to me that other women xx ##xx xx ##xx told me to leave the bank and never come back i told her it was fine as long as i get my cash money back she said no ##o xx ##xx i almost call police f ##ot that finally they return the money but not before the cash ##ier return only xx ##xx instead of xx ##xx my original deposit and it took her another 10 minutes just to count it back [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] for her to train because i know they do nt need me there at all but she refused to give my dl back or make any note just keep trying few more minutes passed i asked one more time and she refused then i tried to grab my dl but she took it faster and step back i was furious then i grab her hand try to pull her so i can get my dl but she still refuse to give it to me that other women xx ##xx xx ##xx told me to leave the bank and never come back i told her it was fine as long as i get my cash money back she said no ##o xx ##xx i almost call police f ##ot that finally they return the money but not before the cash ##ier return only xx ##xx instead of xx ##xx my original deposit and it took her another 10 minutes just to count it back [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 2005 2014 2000 3345 2138 1045 2113 2027 2079 23961 2342 2033 2045 2012 2035 2021 2016 4188 2000 2507 2026 21469 2067 2030 2191 2151 3602 2074 2562 2667 2261 2062 2781 2979 1045 2356 2028 2062 2051 1998 2016 4188 2059 1045 2699 2000 6723 2026 21469 2021 2016 2165 2009 5514 1998 3357 2067 1045 2001 9943 2059 1045 6723 2014 2192 3046 2000 4139 2014 2061 1045 2064 2131 2026 21469 2021 2016 2145 10214 2000 2507 2009 2000 2033 2008 2060 2308 22038 20348 22038 20348 2409 2033 2000 2681 1996 2924 1998 2196 2272 2067 1045 2409 2014 2009 2001 2986 2004 2146 2004 1045 2131 2026 5356 2769 2067 2016 2056 2053 2080 22038 20348 1045 2471 2655 2610 1042 4140 2008 2633 2027 2709 1996 2769 2021 2025 2077 1996 5356 3771 2709 2069 22038 20348 2612 1997 22038 20348 2026 2434 12816 1998 2009 2165 2014 2178 2184 2781 2074 2000 4175 2009 2067 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 2005 2014 2000 3345 2138 1045 2113 2027 2079 23961 2342 2033 2045 2012 2035 2021 2016 4188 2000 2507 2026 21469 2067 2030 2191 2151 3602 2074 2562 2667 2261 2062 2781 2979 1045 2356 2028 2062 2051 1998 2016 4188 2059 1045 2699 2000 6723 2026 21469 2021 2016 2165 2009 5514 1998 3357 2067 1045 2001 9943 2059 1045 6723 2014 2192 3046 2000 4139 2014 2061 1045 2064 2131 2026 21469 2021 2016 2145 10214 2000 2507 2009 2000 2033 2008 2060 2308 22038 20348 22038 20348 2409 2033 2000 2681 1996 2924 1998 2196 2272 2067 1045 2409 2014 2009 2001 2986 2004 2146 2004 1045 2131 2026 5356 2769 2067 2016 2056 2053 2080 22038 20348 1045 2471 2655 2610 1042 4140 2008 2633 2027 2709 1996 2769 2021 2025 2077 1996 5356 3771 2709 2069 22038 20348 2612 1997 22038 20348 2026 2434 12816 1998 2009 2165 2014 2178 2184 2781 2074 2000 4175 2009 2067 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 0 (id = 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 0 (id = 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i have been di ##sp ##uting and requesting validation of a debts i do not owe and basic ina ##cc ##ura ##cies on my credit report for months now and trans ##uni ##on has not only been eva ##sive they have not provided the requested and official documentation required by law nor have they corrected the information in reference to the following accounts since last contact name xx ##xx xx ##xx xx ##xx xx ##xx xx ##xx acc ##t xx ##xx not mine no proof of debt or validation ##xx ##xx xx ##xx acc ##t xx ##xx not mine no proof of debt or validation ##xx ##xx xx ##xx xx ##xx acc ##t xx ##xx not mine no proof of debt or validation ##ina ##cc ##ura ##te inquiries xx ##xx xx ##xx ##xx xx xx ##xx ##xx ##xx xx ##xx ##xx xx xx ##xx ##ple ##ase be advised that this is not a refusal to pay the debt but a observe sent pursuant to the fair debt collection practices act 1 usc 1692 ##g sec 80 ##9 b you claim that i owe this under the fair debt collections practices act i have the right to request validation [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] i have been di ##sp ##uting and requesting validation of a debts i do not owe and basic ina ##cc ##ura ##cies on my credit report for months now and trans ##uni ##on has not only been eva ##sive they have not provided the requested and official documentation required by law nor have they corrected the information in reference to the following accounts since last contact name xx ##xx xx ##xx xx ##xx xx ##xx xx ##xx acc ##t xx ##xx not mine no proof of debt or validation ##xx ##xx xx ##xx acc ##t xx ##xx not mine no proof of debt or validation ##xx ##xx xx ##xx xx ##xx acc ##t xx ##xx not mine no proof of debt or validation ##ina ##cc ##ura ##te inquiries xx ##xx xx ##xx ##xx xx xx ##xx ##xx ##xx xx ##xx ##xx xx xx ##xx ##ple ##ase be advised that this is not a refusal to pay the debt but a observe sent pursuant to the fair debt collection practices act 1 usc 1692 ##g sec 80 ##9 b you claim that i owe this under the fair debt collections practices act i have the right to request validation [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2031 2042 4487 13102 20807 1998 17942 27354 1997 1037 13930 1045 2079 2025 12533 1998 3937 27118 9468 4648 9243 2006 2026 4923 3189 2005 2706 2085 1998 9099 19496 2239 2038 2025 2069 2042 9345 12742 2027 2031 2025 3024 1996 7303 1998 2880 12653 3223 2011 2375 4496 2031 2027 13371 1996 2592 1999 4431 2000 1996 2206 6115 2144 2197 3967 2171 22038 20348 22038 20348 22038 20348 22038 20348 22038 20348 16222 2102 22038 20348 2025 3067 2053 6947 1997 7016 2030 27354 20348 20348 22038 20348 16222 2102 22038 20348 2025 3067 2053 6947 1997 7016 2030 27354 20348 20348 22038 20348 22038 20348 16222 2102 22038 20348 2025 3067 2053 6947 1997 7016 2030 27354 3981 9468 4648 2618 27050 22038 20348 22038 20348 20348 22038 22038 20348 20348 20348 22038 20348 20348 22038 22038 20348 10814 11022 2022 9449 2008 2023 2003 2025 1037 13948 2000 3477 1996 7016 2021 1037 11949 2741 27081 2000 1996 4189 7016 3074 6078 2552 1015 15529 28622 2290 10819 3770 2683 1038 2017 4366 2008 1045 12533 2023 2104 1996 4189 7016 6407 6078 2552 1045 2031 1996 2157 2000 5227 27354 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1045 2031 2042 4487 13102 20807 1998 17942 27354 1997 1037 13930 1045 2079 2025 12533 1998 3937 27118 9468 4648 9243 2006 2026 4923 3189 2005 2706 2085 1998 9099 19496 2239 2038 2025 2069 2042 9345 12742 2027 2031 2025 3024 1996 7303 1998 2880 12653 3223 2011 2375 4496 2031 2027 13371 1996 2592 1999 4431 2000 1996 2206 6115 2144 2197 3967 2171 22038 20348 22038 20348 22038 20348 22038 20348 22038 20348 16222 2102 22038 20348 2025 3067 2053 6947 1997 7016 2030 27354 20348 20348 22038 20348 16222 2102 22038 20348 2025 3067 2053 6947 1997 7016 2030 27354 20348 20348 22038 20348 22038 20348 16222 2102 22038 20348 2025 3067 2053 6947 1997 7016 2030 27354 3981 9468 4648 2618 27050 22038 20348 22038 20348 20348 22038 22038 20348 20348 20348 22038 20348 20348 22038 22038 20348 10814 11022 2022 9449 2008 2023 2003 2025 1037 13948 2000 3477 1996 7016 2021 1037 11949 2741 27081 2000 1996 4189 7016 3074 6078 2552 1015 15529 28622 2290 10819 3770 2683 1038 2017 4366 2008 1045 12533 2023 2104 1996 4189 7016 6407 6078 2552 1045 2031 1996 2157 2000 5227 27354 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] the right to request validation of the debt you say i owe or gave authorization to pull an in ##qui ##re i am requesting proof that i am the accurate party to these or gave authorization to pull my credit and a xx ##xx which is binding on me to pay this debt this is not a request for co ##rro ##bor ##ation via e or evidence of my mail ##ing address but a demand for authentication made pursuant to my name date or birth ss and correct address xx ##xx xx ##xx xx ##xx xx ##xx ky xx ##xx accordance with section of the fair debt collection practices act or i request this be removed as soon as possible i am also requesting the names addresses and telephone numbers of individuals contacted or going to be contacted during this time absent such proof you must correct any er ##rone ##ous reports of these as mine please evidence your authorization under 15 usc 1692 e and 15 usc 1692 f in this alleged matter what is your authorization of law for your collection of information what is your authorization of law for your collection of this alleged debt [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] the right to request validation of the debt you say i owe or gave authorization to pull an in ##qui ##re i am requesting proof that i am the accurate party to these or gave authorization to pull my credit and a xx ##xx which is binding on me to pay this debt this is not a request for co ##rro ##bor ##ation via e or evidence of my mail ##ing address but a demand for authentication made pursuant to my name date or birth ss and correct address xx ##xx xx ##xx xx ##xx xx ##xx ky xx ##xx accordance with section of the fair debt collection practices act or i request this be removed as soon as possible i am also requesting the names addresses and telephone numbers of individuals contacted or going to be contacted during this time absent such proof you must correct any er ##rone ##ous reports of these as mine please evidence your authorization under 15 usc 1692 e and 15 usc 1692 f in this alleged matter what is your authorization of law for your collection of information what is your authorization of law for your collection of this alleged debt [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1996 2157 2000 5227 27354 1997 1996 7016 2017 2360 1045 12533 2030 2435 20104 2000 4139 2019 1999 15549 2890 1045 2572 17942 6947 2008 1045 2572 1996 8321 2283 2000 2122 2030 2435 20104 2000 4139 2026 4923 1998 1037 22038 20348 2029 2003 8031 2006 2033 2000 3477 2023 7016 2023 2003 2025 1037 5227 2005 2522 18933 12821 3370 3081 1041 2030 3350 1997 2026 5653 2075 4769 2021 1037 5157 2005 27280 2081 27081 2000 2026 2171 3058 2030 4182 7020 1998 6149 4769 22038 20348 22038 20348 22038 20348 22038 20348 18712 22038 20348 10388 2007 2930 1997 1996 4189 7016 3074 6078 2552 2030 1045 5227 2023 2022 3718 2004 2574 2004 2825 1045 2572 2036 17942 1996 3415 11596 1998 7026 3616 1997 3633 11925 2030 2183 2000 2022 11925 2076 2023 2051 9962 2107 6947 2017 2442 6149 2151 9413 20793 3560 4311 1997 2122 2004 3067 3531 3350 2115 20104 2104 2321 15529 28622 1041 1998 2321 15529 28622 1042 1999 2023 6884 3043 2054 2003 2115 20104 1997 2375 2005 2115 3074 1997 2592 2054 2003 2115 20104 1997 2375 2005 2115 3074 1997 2023 6884 7016 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 1996 2157 2000 5227 27354 1997 1996 7016 2017 2360 1045 12533 2030 2435 20104 2000 4139 2019 1999 15549 2890 1045 2572 17942 6947 2008 1045 2572 1996 8321 2283 2000 2122 2030 2435 20104 2000 4139 2026 4923 1998 1037 22038 20348 2029 2003 8031 2006 2033 2000 3477 2023 7016 2023 2003 2025 1037 5227 2005 2522 18933 12821 3370 3081 1041 2030 3350 1997 2026 5653 2075 4769 2021 1037 5157 2005 27280 2081 27081 2000 2026 2171 3058 2030 4182 7020 1998 6149 4769 22038 20348 22038 20348 22038 20348 22038 20348 18712 22038 20348 10388 2007 2930 1997 1996 4189 7016 3074 6078 2552 2030 1045 5227 2023 2022 3718 2004 2574 2004 2825 1045 2572 2036 17942 1996 3415 11596 1998 7026 3616 1997 3633 11925 2030 2183 2000 2022 11925 2076 2023 2051 9962 2107 6947 2017 2442 6149 2151 9413 20793 3560 4311 1997 2122 2004 3067 3531 3350 2115 20104 2104 2321 15529 28622 1041 1998 2321 15529 28622 1042 1999 2023 6884 3043 2054 2003 2115 20104 1997 2375 2005 2115 3074 1997 2592 2054 2003 2115 20104 1997 2375 2005 2115 3074 1997 2023 6884 7016 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Example ***\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:guid: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] my complaint is against the credit bureau e ##qui ##fa ##x several attempts have been made to all three credit bureau ##s in an attempt to have a negative item removed e ##qui ##fa ##x has repeatedly failed to properly investigate my dispute and i have provided more than enough documentation to prove me claim is true and accurate had e ##qui ##fa ##x seriously investigated my claim they would have found the proof of the error is in the credit report itself in my credit report in the closed account section there is an account with xx ##xx s xx ##xx xx ##xx the information states account open xx ##xx xx ##xx xx ##xx last payment xx ##xx xx ##xx xx ##xx closed on xx ##xx xx ##xx xx ##xx xx ##xx xx ##xx xx ##xx acquired the debt at some point and fraudulent ##ly re aged the debt on the credit report they have falsely reported the following account xx ##xx xx ##xx xx ##xx date open xx ##xx xx ##xx xx ##xx date of first del ##ique ##ncy xx ##xx date of first major del ##ique ##ncy xx ##xx i received notice from e ##qui [SEP]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:tokens: [CLS] my complaint is against the credit bureau e ##qui ##fa ##x several attempts have been made to all three credit bureau ##s in an attempt to have a negative item removed e ##qui ##fa ##x has repeatedly failed to properly investigate my dispute and i have provided more than enough documentation to prove me claim is true and accurate had e ##qui ##fa ##x seriously investigated my claim they would have found the proof of the error is in the credit report itself in my credit report in the closed account section there is an account with xx ##xx s xx ##xx xx ##xx the information states account open xx ##xx xx ##xx xx ##xx last payment xx ##xx xx ##xx xx ##xx closed on xx ##xx xx ##xx xx ##xx xx ##xx xx ##xx xx ##xx acquired the debt at some point and fraudulent ##ly re aged the debt on the credit report they have falsely reported the following account xx ##xx xx ##xx xx ##xx date open xx ##xx xx ##xx xx ##xx date of first del ##ique ##ncy xx ##xx date of first major del ##ique ##ncy xx ##xx i received notice from e ##qui [SEP]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 2026 12087 2003 2114 1996 4923 4879 1041 15549 7011 2595 2195 4740 2031 2042 2081 2000 2035 2093 4923 4879 2015 1999 2019 3535 2000 2031 1037 4997 8875 3718 1041 15549 7011 2595 2038 8385 3478 2000 7919 8556 2026 7593 1998 1045 2031 3024 2062 2084 2438 12653 2000 6011 2033 4366 2003 2995 1998 8321 2018 1041 15549 7011 2595 5667 10847 2026 4366 2027 2052 2031 2179 1996 6947 1997 1996 7561 2003 1999 1996 4923 3189 2993 1999 2026 4923 3189 1999 1996 2701 4070 2930 2045 2003 2019 4070 2007 22038 20348 1055 22038 20348 22038 20348 1996 2592 2163 4070 2330 22038 20348 22038 20348 22038 20348 2197 7909 22038 20348 22038 20348 22038 20348 2701 2006 22038 20348 22038 20348 22038 20348 22038 20348 22038 20348 22038 20348 3734 1996 7016 2012 2070 2391 1998 27105 2135 2128 4793 1996 7016 2006 1996 4923 3189 2027 2031 23123 2988 1996 2206 4070 22038 20348 22038 20348 22038 20348 3058 2330 22038 20348 22038 20348 22038 20348 3058 1997 2034 3972 7413 9407 22038 20348 3058 1997 2034 2350 3972 7413 9407 22038 20348 1045 2363 5060 2013 1041 15549 102\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_ids: 101 2026 12087 2003 2114 1996 4923 4879 1041 15549 7011 2595 2195 4740 2031 2042 2081 2000 2035 2093 4923 4879 2015 1999 2019 3535 2000 2031 1037 4997 8875 3718 1041 15549 7011 2595 2038 8385 3478 2000 7919 8556 2026 7593 1998 1045 2031 3024 2062 2084 2438 12653 2000 6011 2033 4366 2003 2995 1998 8321 2018 1041 15549 7011 2595 5667 10847 2026 4366 2027 2052 2031 2179 1996 6947 1997 1996 7561 2003 1999 1996 4923 3189 2993 1999 2026 4923 3189 1999 1996 2701 4070 2930 2045 2003 2019 4070 2007 22038 20348 1055 22038 20348 22038 20348 1996 2592 2163 4070 2330 22038 20348 22038 20348 22038 20348 2197 7909 22038 20348 22038 20348 22038 20348 2701 2006 22038 20348 22038 20348 22038 20348 22038 20348 22038 20348 22038 20348 3734 1996 7016 2012 2070 2391 1998 27105 2135 2128 4793 1996 7016 2006 1996 4923 3189 2027 2031 23123 2988 1996 2206 4070 22038 20348 22038 20348 22038 20348 3058 2330 22038 20348 22038 20348 22038 20348 3058 1997 2034 3972 7413 9407 22038 20348 3058 1997 2034 2350 3972 7413 9407 22038 20348 1045 2363 5060 2013 1041 15549 102\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:label: 3 (id = 3)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLkn7RQ8_CV1",
        "colab_type": "code",
        "outputId": "6de0cd1c-24c1-4cf1-beca-3e02941a8c82",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "#Example on first observation in the training set\n",
        "print(\"Sentence : \", train_InputExamples.iloc[0].text_a)\n",
        "print(\"-\"*30)\n",
        "print(\"Tokens : \", tokenizer.tokenize(train_InputExamples.iloc[0].text_a))\n",
        "print(\"-\"*30)\n",
        "print(\"Input IDs : \", train_features[0].input_ids)\n",
        "print(\"-\"*30)\n",
        "print(\"Input Masks : \", train_features[0].input_mask)\n",
        "print(\"-\"*30)\n",
        "print(\"Segment IDs : \", train_features[0].segment_ids)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentence :  I was sold a Timeshare in XXXX XXXX NJ at the XXXX XXXX through a company doing business as XXXX Subsequent to the agreement I had an issue where my wife took ill and needed surgery Since she is the primary income provider I found myself unable to meet all of my obligations I determined the Timeshare was an unneeded expense so they were not paid They started harassing me I looked into this whole timeshare issue and found out that my membership was nothing more than a source of income for the Hotel that if I were to simply reserve a room through the Internet at the exact same location it would cost me less than HALF what I pay annually in Maintenance fees and taxes When they called back after I discovered this I informed them that I was misled and refused to pay any more into this unless they could prove to me any sort of value in what I purchased They have since then threatened me with legal action including Check Fraud and when I informed them that their threats were in fact a violation of the Fair Debt Collection Act they stopped the calls but\n",
            "------------------------------\n",
            "Tokens :  ['i', 'was', 'sold', 'a', 'times', '##har', '##e', 'in', 'xx', '##xx', 'xx', '##xx', 'nj', 'at', 'the', 'xx', '##xx', 'xx', '##xx', 'through', 'a', 'company', 'doing', 'business', 'as', 'xx', '##xx', 'subsequent', 'to', 'the', 'agreement', 'i', 'had', 'an', 'issue', 'where', 'my', 'wife', 'took', 'ill', 'and', 'needed', 'surgery', 'since', 'she', 'is', 'the', 'primary', 'income', 'provider', 'i', 'found', 'myself', 'unable', 'to', 'meet', 'all', 'of', 'my', 'obligations', 'i', 'determined', 'the', 'times', '##har', '##e', 'was', 'an', 'un', '##nee', '##ded', 'expense', 'so', 'they', 'were', 'not', 'paid', 'they', 'started', 'hara', '##ssing', 'me', 'i', 'looked', 'into', 'this', 'whole', 'times', '##har', '##e', 'issue', 'and', 'found', 'out', 'that', 'my', 'membership', 'was', 'nothing', 'more', 'than', 'a', 'source', 'of', 'income', 'for', 'the', 'hotel', 'that', 'if', 'i', 'were', 'to', 'simply', 'reserve', 'a', 'room', 'through', 'the', 'internet', 'at', 'the', 'exact', 'same', 'location', 'it', 'would', 'cost', 'me', 'less', 'than', 'half', 'what', 'i', 'pay', 'annually', 'in', 'maintenance', 'fees', 'and', 'taxes', 'when', 'they', 'called', 'back', 'after', 'i', 'discovered', 'this', 'i', 'informed', 'them', 'that', 'i', 'was', 'mis', '##led', 'and', 'refused', 'to', 'pay', 'any', 'more', 'into', 'this', 'unless', 'they', 'could', 'prove', 'to', 'me', 'any', 'sort', 'of', 'value', 'in', 'what', 'i', 'purchased', 'they', 'have', 'since', 'then', 'threatened', 'me', 'with', 'legal', 'action', 'including', 'check', 'fraud', 'and', 'when', 'i', 'informed', 'them', 'that', 'their', 'threats', 'were', 'in', 'fact', 'a', 'violation', 'of', 'the', 'fair', 'debt', 'collection', 'act', 'they', 'stopped', 'the', 'calls', 'but']\n",
            "------------------------------\n",
            "Input IDs :  [101, 1045, 2001, 2853, 1037, 2335, 8167, 2063, 1999, 22038, 20348, 22038, 20348, 19193, 2012, 1996, 22038, 20348, 22038, 20348, 2083, 1037, 2194, 2725, 2449, 2004, 22038, 20348, 4745, 2000, 1996, 3820, 1045, 2018, 2019, 3277, 2073, 2026, 2564, 2165, 5665, 1998, 2734, 5970, 2144, 2016, 2003, 1996, 3078, 3318, 10802, 1045, 2179, 2870, 4039, 2000, 3113, 2035, 1997, 2026, 14422, 1045, 4340, 1996, 2335, 8167, 2063, 2001, 2019, 4895, 24045, 5732, 10961, 2061, 2027, 2020, 2025, 3825, 2027, 2318, 18820, 18965, 2033, 1045, 2246, 2046, 2023, 2878, 2335, 8167, 2063, 3277, 1998, 2179, 2041, 2008, 2026, 5779, 2001, 2498, 2062, 2084, 1037, 3120, 1997, 3318, 2005, 1996, 3309, 2008, 2065, 1045, 2020, 2000, 3432, 3914, 1037, 2282, 2083, 1996, 4274, 2012, 1996, 6635, 2168, 3295, 2009, 2052, 3465, 2033, 2625, 2084, 2431, 2054, 1045, 3477, 6604, 1999, 6032, 9883, 1998, 7773, 2043, 2027, 2170, 2067, 2044, 1045, 3603, 2023, 1045, 6727, 2068, 2008, 1045, 2001, 28616, 3709, 1998, 4188, 2000, 3477, 2151, 2062, 2046, 2023, 4983, 2027, 2071, 6011, 2000, 2033, 2151, 4066, 1997, 3643, 1999, 2054, 1045, 4156, 2027, 2031, 2144, 2059, 5561, 2033, 2007, 3423, 2895, 2164, 4638, 9861, 1998, 2043, 1045, 6727, 2068, 2008, 2037, 102]\n",
            "------------------------------\n",
            "Input Masks :  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
            "------------------------------\n",
            "Segment IDs :  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "piypsrPudMFf",
        "colab_type": "text"
      },
      "source": [
        "# BERT: Creating A Multi-Class Classifier Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBxxy9s7GCW4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_model(is_predicting, input_ids, input_mask, segment_ids, labels,\n",
        "                 num_labels):\n",
        "  \n",
        "  bert_module = hub.Module(\n",
        "      BERT_MODEL_HUB,\n",
        "      trainable=True)\n",
        "  bert_inputs = dict(\n",
        "      input_ids=input_ids,\n",
        "      input_mask=input_mask,\n",
        "      segment_ids=segment_ids)\n",
        "  bert_outputs = bert_module(\n",
        "      inputs=bert_inputs,\n",
        "      signature=\"tokens\",\n",
        "      as_dict=True)\n",
        "\n",
        "  # Use \"pooled_output\" for classification tasks on an entire sentence.\n",
        "  # Use \"sequence_outputs\" for token-level output.\n",
        "  output_layer = bert_outputs[\"pooled_output\"]\n",
        "  # with tf.Session() as sess:\n",
        "  output_layer1 = bert_outputs[\"pooled_output\"]\n",
        "  # output_layer1 = 999\n",
        "  hidden_size = output_layer.shape[-1].value\n",
        "\n",
        "  # Create our own layer to tune for politeness data.\n",
        "  output_weights = tf.get_variable(\n",
        "      \"output_weights\", [num_labels, hidden_size],\n",
        "      initializer=tf.truncated_normal_initializer(stddev=0.02))\n",
        "\n",
        "  output_bias = tf.get_variable(\n",
        "      \"output_bias\", [num_labels], initializer=tf.zeros_initializer())\n",
        "\n",
        "  with tf.variable_scope(\"loss\"):\n",
        "\n",
        "    # Dropout helps prevent overfitting\n",
        "    output_layer = tf.nn.dropout(output_layer, keep_prob=0.8)\n",
        "\n",
        "    logits = tf.matmul(output_layer, output_weights, transpose_b=True)\n",
        "    logits = tf.nn.bias_add(logits, output_bias)\n",
        "    log_probs = tf.nn.log_softmax(logits, axis=-1)\n",
        "\n",
        "    # Convert labels into one-hot encoding\n",
        "    one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)\n",
        "\n",
        "    predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))\n",
        "    # If we're predicting, we want predicted labels and the probabiltiies.\n",
        "    if is_predicting:\n",
        "      return (predicted_labels, log_probs, output_layer1)\n",
        "\n",
        "    # If we're train/eval, compute loss between predicted and actual label\n",
        "    per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)\n",
        "    loss = tf.reduce_mean(per_example_loss)\n",
        "    return (loss, predicted_labels, log_probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rPRB5i1HG8JO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def model_fn_builder(num_labels, learning_rate, num_train_steps,\n",
        "                     num_warmup_steps):\n",
        "  \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n",
        "  def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument\n",
        "    \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n",
        "\n",
        "    input_ids = features[\"input_ids\"]\n",
        "    input_mask = features[\"input_mask\"]\n",
        "    segment_ids = features[\"segment_ids\"]\n",
        "    label_ids = features[\"label_ids\"]\n",
        "\n",
        "    is_predicting = (mode == tf.estimator.ModeKeys.PREDICT)\n",
        "    \n",
        "    # TRAIN and EVAL\n",
        "    if not is_predicting:\n",
        "\n",
        "      (loss, predicted_labels, log_probs) = create_model(\n",
        "        is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n",
        "\n",
        "      train_op = bert.optimization.create_optimizer(\n",
        "          loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)\n",
        "\n",
        "      # Calculate evaluation metrics. \n",
        "      def metric_fn(label_ids, predicted_labels):\n",
        "        accuracy = tf.metrics.accuracy(label_ids, predicted_labels)\n",
        "        true_pos = tf.metrics.true_positives(\n",
        "            label_ids,\n",
        "            predicted_labels)\n",
        "        true_neg = tf.metrics.true_negatives(\n",
        "            label_ids,\n",
        "            predicted_labels)   \n",
        "        false_pos = tf.metrics.false_positives(\n",
        "            label_ids,\n",
        "            predicted_labels)  \n",
        "        false_neg = tf.metrics.false_negatives(\n",
        "            label_ids,\n",
        "            predicted_labels)\n",
        "        \n",
        "        return {\n",
        "            \"eval_accuracy\": accuracy,\n",
        "            \"true_positives\": true_pos,\n",
        "            \"true_negatives\": true_neg,\n",
        "            \"false_positives\": false_pos,\n",
        "            \"false_negatives\": false_neg,\n",
        "            }\n",
        "\n",
        "      eval_metrics = metric_fn(label_ids, predicted_labels)\n",
        "\n",
        "      if mode == tf.estimator.ModeKeys.TRAIN:\n",
        "        return tf.estimator.EstimatorSpec(mode=mode,\n",
        "          loss=loss,\n",
        "          train_op=train_op)\n",
        "      else:\n",
        "          return tf.estimator.EstimatorSpec(mode=mode,\n",
        "            loss=loss,\n",
        "            eval_metric_ops=eval_metrics)\n",
        "    else:\n",
        "      (predicted_labels, log_probs, output_layer) = create_model(\n",
        "        is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n",
        "      predictions = {\n",
        "          'probabilities': log_probs,\n",
        "          'labels': predicted_labels,\n",
        "          'pooled_output': output_layer\n",
        "      }\n",
        "      return tf.estimator.EstimatorSpec(mode, predictions=predictions)\n",
        "\n",
        "  # Return the actual model function in the closure\n",
        "  return model_fn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fNrvabUFHC79",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 16\n",
        "LEARNING_RATE = 2e-5\n",
        "NUM_TRAIN_EPOCHS = 1.0\n",
        "# Warmup is a period of time where the learning rate is small and gradually increases--usually helps training.\n",
        "WARMUP_PROPORTION = 0.1\n",
        "# Model configs\n",
        "SAVE_CHECKPOINTS_STEPS = 300\n",
        "SAVE_SUMMARY_STEPS = 100\n",
        "\n",
        "# Compute train and warmup steps from batch size\n",
        "num_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)\n",
        "num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)\n",
        "\n",
        "# Specify output directory and number of checkpoint steps to save\n",
        "run_config = tf.estimator.RunConfig(\n",
        "    model_dir=OUTPUT_DIR,\n",
        "    save_summary_steps=SAVE_SUMMARY_STEPS,\n",
        "    save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)\n",
        "\n",
        "# Specify output directory and number of checkpoint steps to save\n",
        "run_config = tf.estimator.RunConfig(\n",
        "    model_dir=OUTPUT_DIR,\n",
        "    save_summary_steps=SAVE_SUMMARY_STEPS,\n",
        "    save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xo70COnsHIWE",
        "colab_type": "code",
        "outputId": "febad872-9d36-443b-b7ed-9202903facae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "num_train_steps, len(label_list)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1979, 10)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zHlZKcq7HMzE",
        "colab_type": "code",
        "outputId": "9c5ee0bf-d111-48f2-be4d-c01362521b6e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        }
      },
      "source": [
        "#Initializing the model and the estimator\n",
        "model_fn = model_fn_builder(\n",
        "  num_labels=len(label_list),\n",
        "  learning_rate=LEARNING_RATE,\n",
        "  num_train_steps=num_train_steps,\n",
        "  num_warmup_steps=num_warmup_steps)\n",
        "\n",
        "estimator = tf.estimator.Estimator(\n",
        "  model_fn=model_fn,\n",
        "  config=run_config,\n",
        "  params={\"batch_size\": BATCH_SIZE})\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using config: {'_model_dir': '/bert_news_category', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 300, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            "graph_options {\n",
            "  rewrite_options {\n",
            "    meta_optimizer_iterations: ONE\n",
            "  }\n",
            "}\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f1fc8615630>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using config: {'_model_dir': '/bert_news_category', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 300, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
            "graph_options {\n",
            "  rewrite_options {\n",
            "    meta_optimizer_iterations: ONE\n",
            "  }\n",
            "}\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f1fc8615630>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "05KBWhqeHUIF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create an input function for training. drop_remainder = True for using TPUs.\n",
        "train_input_fn = bert.run_classifier.input_fn_builder(\n",
        "    features=train_features,\n",
        "    seq_length=MAX_SEQ_LENGTH,\n",
        "    is_training=True,\n",
        "    drop_remainder=False)\n",
        "\n",
        "# Create an input function for validating. drop_remainder = True for using TPUs.\n",
        "val_input_fn = run_classifier.input_fn_builder(\n",
        "    features=val_features,\n",
        "    seq_length=MAX_SEQ_LENGTH,\n",
        "    is_training=False,\n",
        "    drop_remainder=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lm0AubBnhEst",
        "colab_type": "text"
      },
      "source": [
        "# BERT: Fine Tuning Training & Evaluating"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JUhCik-iHj9o",
        "colab_type": "code",
        "outputId": "6a44d761-e385-415f-bdc6-343c038adf67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "#Training the model\n",
        "print(f'Beginning Training!')\n",
        "current_time = datetime.now()\n",
        "estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\n",
        "print(\"Training took time \", datetime.now() - current_time)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Beginning Training!\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Calling model_fn.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Calling model_fn.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-52-a75d089b65af>:35: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-52-a75d089b65af>:35: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/bert/optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/math_grad.py:1375: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/framework/indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Done calling model_fn.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Done calling model_fn.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Create CheckpointSaverHook.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Create CheckpointSaverHook.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Graph was finalized.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Graph was finalized.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Running local_init_op.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Running local_init_op.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Done running local_init_op.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Done running local_init_op.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 0 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 0 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 2.4308736, step = 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 2.4308736, step = 0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 1.73687\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 1.73687\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.9394433, step = 100 (57.578 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.9394433, step = 100 (57.578 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46178\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46178\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.5528544, step = 200 (40.620 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.5528544, step = 200 (40.620 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 300 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 300 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.17765\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.17765\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.71076536, step = 300 (45.921 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.71076536, step = 300 (45.921 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46341\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46341\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.96843326, step = 400 (40.594 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.96843326, step = 400 (40.594 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46412\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46412\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.6834893, step = 500 (40.582 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.6834893, step = 500 (40.582 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 600 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 600 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.18593\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.18593\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.57014453, step = 600 (45.747 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.57014453, step = 600 (45.747 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46392\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46392\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.18577647, step = 700 (40.586 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.18577647, step = 700 (40.586 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46287\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46287\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.44487035, step = 800 (40.603 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.44487035, step = 800 (40.603 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 900 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 900 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.17444\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.17444\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.91815966, step = 900 (45.989 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.91815966, step = 900 (45.989 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46392\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46392\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.96589124, step = 1000 (40.585 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.96589124, step = 1000 (40.585 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.464\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.464\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.52840304, step = 1100 (40.585 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.52840304, step = 1100 (40.585 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1200 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1200 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.17033\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.17033\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.53256005, step = 1200 (46.075 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.53256005, step = 1200 (46.075 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.4627\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.4627\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.21745712, step = 1300 (40.606 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.21745712, step = 1300 (40.606 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46223\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46223\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.781665, step = 1400 (40.613 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.781665, step = 1400 (40.613 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1500 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1500 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use standard file APIs to delete files with this prefix.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use standard file APIs to delete files with this prefix.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.16825\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.16825\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.3196971, step = 1500 (46.120 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.3196971, step = 1500 (46.120 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46084\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46084\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.28183633, step = 1600 (40.637 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.28183633, step = 1600 (40.637 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46311\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46311\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.50631773, step = 1700 (40.598 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.50631773, step = 1700 (40.598 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1800 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1800 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.16985\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.16985\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.5347062, step = 1800 (46.086 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.5347062, step = 1800 (46.086 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46108\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:global_step/sec: 2.46108\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.54360807, step = 1900 (40.633 sec)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:loss = 0.54360807, step = 1900 (40.633 sec)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1979 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saving checkpoints for 1979 into /bert_news_category/model.ckpt.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Loss for final step: 0.2288338.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Loss for final step: 0.2288338.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training took time  0:15:36.286117\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b-EFK3DS0qkm",
        "colab_type": "text"
      },
      "source": [
        "The accuracy for the fine tuned BERT model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HSQxiqDPHrJy",
        "colab_type": "code",
        "outputId": "c9705bcc-c7c3-41fa-99e2-0f6e8f446160",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "#Evaluating the model with Validation set\n",
        "estimator.evaluate(input_fn=val_input_fn, steps=None)"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/framework/indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'eval_accuracy': 0.8582935,\n",
              " 'false_negatives': 116.0,\n",
              " 'false_positives': 176.0,\n",
              " 'global_step': 1979,\n",
              " 'loss': 0.4725039,\n",
              " 'true_negatives': 693.0,\n",
              " 'true_positives': 6961.0}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ubX4mo7ahbZI",
        "colab_type": "text"
      },
      "source": [
        "# BERT: Get The Vector Transformations from the Fine Tuned BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7B114QMlVwMm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# A method to get predictions\n",
        "def getPrediction(in_sentences, type_output = \"features\"):\n",
        "  #A list to map the actual labels to the predictions\n",
        "  labels = np.unique(train['label'])\n",
        "  input_examples = [run_classifier.InputExample(guid=\"\", text_a = x, text_b = None, label = 0) for x in in_sentences] \n",
        "  input_features = run_classifier.convert_examples_to_features(input_examples, label_list, MAX_SEQ_LENGTH, tokenizer)\n",
        "  #Predicting the classes \n",
        "  predict_input_fn = run_classifier.input_fn_builder(features=input_features, seq_length=MAX_SEQ_LENGTH, is_training=False, drop_remainder=False)\n",
        "  predictions = estimator.predict(predict_input_fn)\n",
        "  if type_output == \"features\":\n",
        "    return [prediction['pooled_output'] for _,prediction in enumerate(predictions) ]\n",
        "  else:\n",
        "    return ([(sentence, prediction['probabilities'],\n",
        "              prediction['labels'], labels[prediction['labels']]) for sentence, prediction in zip(in_sentences, predictions)])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WKpDMg-nV-yZ",
        "colab_type": "code",
        "outputId": "af7b21e9-a87c-4de6-8168-6109991f4154",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tf.compat.v1.logging.set_verbosity(tf.logging.ERROR)\n",
        "MAX_SEQ_LENGTH"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "200"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjRdLdqj-mpo",
        "colab_type": "code",
        "outputId": "4f388f8d-e19b-42fa-fae2-74ce196c37ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_df.shape, val_df.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((31679, 2), (7946, 2))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m4Q7Ih3nmNXh",
        "colab_type": "text"
      },
      "source": [
        "Now extracting the representations:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2z3hwrsaWECM",
        "colab_type": "code",
        "outputId": "77174d69-c6ce-496d-d0c0-de79357db931",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "tr_emb = np.apply_along_axis(getPrediction, 0,np.array(train_df[DATA_COLUMN]))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 2min 35s, sys: 10.9 s, total: 2min 46s\n",
            "Wall time: 6min 10s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bw6nDeP2WR_u",
        "colab_type": "code",
        "outputId": "fa1031d3-00f3-443d-ff63-60234d0c9375",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%%time\n",
        "val_emb = np.apply_along_axis(getPrediction, 0,np.array(val_df[DATA_COLUMN]))\n",
        "val_emb.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 43.4 s, sys: 2.76 s, total: 46.2 s\n",
            "Wall time: 1min 37s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQbVWlptWUGE",
        "colab_type": "code",
        "outputId": "1eca64c4-d696-4f8b-e154-e2d4bb8f1b2b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "val_emb.shape, tr_emb.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((7946, 768), (31679, 768))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AbIDKTbw8lOt",
        "colab_type": "text"
      },
      "source": [
        "and make the dataset for train and val:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4bBZiJG6hEdU",
        "colab_type": "code",
        "outputId": "24cdb344-81c4-4ebc-8967-cc8c746fa40c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "aux = -1\n",
        "len_l = 0\n",
        "train_x = {}\n",
        "for l, emb in zip(index_l, tr_emb):\n",
        "  if l in train_x.keys():\n",
        "    train_x[l]  =np.vstack([train_x[l], emb])\n",
        "  else:\n",
        "    train_x[l] = [emb]\n",
        "\n",
        "len(train_x.keys())\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "13713"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oq2tpvrUkyoa",
        "colab_type": "code",
        "outputId": "f04c8fd6-c478-4cb3-a6d9-a3853a2b841c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "train_l_final = []\n",
        "label_l_final = []\n",
        "for k in train_x.keys():\n",
        "  train_l_final.append(train_x[k])\n",
        "  label_l_final.append(train.loc[k]['label'])\n",
        "\n",
        "df_train = pd.DataFrame({'emb': train_l_final, 'label': label_l_final, })\n",
        "df_train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>emb</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[[0.848892, 0.05472551, -0.85581946, 0.4909455...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[[0.4019187, 0.30476514, 0.034283057, -0.10456...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[[0.9221226, -0.48432896, -0.9658792, 0.905419...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[[-0.34883273, 0.5719657, -0.65025216, 0.69365...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[[0.8565562, -0.18846692, -0.98491526, 0.75137...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 emb  label\n",
              "0  [[0.848892, 0.05472551, -0.85581946, 0.4909455...      3\n",
              "1  [[0.4019187, 0.30476514, 0.034283057, -0.10456...      4\n",
              "2  [[0.9221226, -0.48432896, -0.9658792, 0.905419...      0\n",
              "3  [[-0.34883273, 0.5719657, -0.65025216, 0.69365...      3\n",
              "4  [[0.8565562, -0.18846692, -0.98491526, 0.75137...      2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cnjcaZHqk6rf",
        "colab_type": "code",
        "outputId": "b66be414-0f35-484f-a3e7-5b5b513d3e67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "aux = -1\n",
        "len_l = 0\n",
        "val_x = {}\n",
        "\n",
        "for l, emb in zip(val_index_l, val_emb):\n",
        "  if l in val_x.keys():\n",
        "    val_x[l]  =np.vstack([val_x[l], emb])\n",
        "  else:\n",
        "    val_x[l] = [emb]\n",
        "\n",
        "\n",
        "val_l_final = []\n",
        "vlabel_l_final = []\n",
        "for k in val_x.keys():\n",
        "  val_l_final.append(val_x[k])\n",
        "  vlabel_l_final.append(val.loc[k]['label'])\n",
        "\n",
        "df_val = pd.DataFrame({'emb': val_l_final, 'label': vlabel_l_final})\n",
        "df_val.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>emb</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[[0.7906871, -0.3191274, -0.82637614, 0.760878...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[[-0.18997923, 0.5434573, -0.8482759, 0.609339...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[[-0.2863525, 0.47603318, -0.84779084, 0.70149...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[[-0.55615234, 0.6234803, -0.7726333, -0.04120...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[[0.24671736, 0.24032024, 0.47350714, -0.00336...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 emb  label\n",
              "0  [[0.7906871, -0.3191274, -0.82637614, 0.760878...      0\n",
              "1  [[-0.18997923, 0.5434573, -0.8482759, 0.609339...      3\n",
              "2  [[-0.2863525, 0.47603318, -0.84779084, 0.70149...      3\n",
              "3  [[-0.55615234, 0.6234803, -0.7726333, -0.04120...      3\n",
              "4  [[0.24671736, 0.24032024, 0.47350714, -0.00336...      2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VMDe3KvcIM5q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_val, df_test = train_test_split(df_val, test_size=0.4, random_state=35)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rzIznwgQiD6x",
        "colab_type": "text"
      },
      "source": [
        "# LSTM: Creating the Final Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "260A5YvElD2D",
        "colab_type": "code",
        "outputId": "f57ea208-93e2-460f-c40f-eff2cddf84a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "source": [
        "from keras import layers\n",
        "text_input = Input(shape=(None,768,), dtype='float32', name='text')\n",
        "\n",
        "l_mask = layers.Masking(mask_value=-99.)(text_input)\n",
        "# Which we encoded in a single vector via a LSTM\n",
        "encoded_text = layers.LSTM(100,)(l_mask)\n",
        "out_dense = layers.Dense(30, activation='relu')(encoded_text)\n",
        "# And we add a softmax classifier on top\n",
        "out = layers.Dense(len(label_list), activation='softmax')(out_dense)\n",
        "# At model instantiation, we specify the input and the output:\n",
        "model = Model(text_input, out)\n",
        "model.compile(optimizer='adam',\n",
        "              loss='sparse_categorical_crossentropy',\n",
        "              metrics=['acc'])\n",
        "model.summary()"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "text (InputLayer)            (None, None, 768)         0         \n",
            "_________________________________________________________________\n",
            "masking_2 (Masking)          (None, None, 768)         0         \n",
            "_________________________________________________________________\n",
            "lstm_2 (LSTM)                (None, 100)               347600    \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 30)                3030      \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 10)                310       \n",
            "=================================================================\n",
            "Total params: 350,940\n",
            "Trainable params: 350,940\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_SR7cUPRlvNg",
        "colab_type": "code",
        "outputId": "da689c92-5825-4297-9448-051def60e82b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_train.shape, df_val.shape, df_test.shape"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((13713, 2), (2057, 2), (1372, 2))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3z6awGncq9wB",
        "colab_type": "text"
      },
      "source": [
        "The generator functions:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5vAf9GmGlSnm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_sequences = len(df_train['emb'].to_list())\n",
        "batch_size = 3\n",
        "batches_per_epoch =  4571\n",
        "assert batch_size * batches_per_epoch == num_sequences\n",
        "num_features= 768\n",
        "def train_generator(df):\n",
        "    x_list= df['emb'].to_list()\n",
        "    y_list =  df.label.to_list()\n",
        "    # Generate batches\n",
        "    while True:\n",
        "        for b in range(batches_per_epoch):\n",
        "            longest_index = (b + 1) * batch_size - 1\n",
        "            timesteps = len(max(df['emb'].to_list()[:(b + 1) * batch_size][-batch_size:], key=len))\n",
        "            x_train = np.full((batch_size, timesteps, num_features), -99.)\n",
        "            y_train = np.zeros((batch_size,  1))\n",
        "            for i in range(batch_size):\n",
        "                li = b * batch_size + i\n",
        "                x_train[i, 0:len(x_list[li]), :] = x_list[li]\n",
        "                y_train[i] = y_list[li]\n",
        "            yield x_train, y_train"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ezFSiXl_meEo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_sequences_val = len(df_val['emb'].to_list())\n",
        "batch_size_val = 11\n",
        "batches_per_epoch_val = 187\n",
        "assert batch_size_val * batches_per_epoch_val == num_sequences_val\n",
        "num_features= 768\n",
        "def val_generator(df):\n",
        "    x_list= df['emb'].to_list()\n",
        "    y_list =  df.label.to_list()\n",
        "    # Generate batches\n",
        "    while True:\n",
        "        for b in range(batches_per_epoch_val):\n",
        "            longest_index = (b + 1) * batch_size_val - 1\n",
        "            timesteps = len(max(df['emb'].to_list()[:(b + 1) * batch_size_val][-31:], key=len))\n",
        "            # print(len(df_train['emb'].to_list()[:b+batch_size][-7:]))\n",
        "            x_train = np.full((batch_size_val, timesteps, num_features), -99.)\n",
        "            y_train = np.zeros((batch_size_val,  1))\n",
        "            for i in range(batch_size_val):\n",
        "                li = b * batch_size_val + i\n",
        "                # print(\"li\", li)\n",
        "                # print(x_train[i, 0:len(x_list[li]), :].shape, len(x_list[li]))\n",
        "                x_train[i, 0:len(x_list[li]), :] = x_list[li]\n",
        "                y_train[i] = y_list[li]\n",
        "            yield x_train, y_train"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jVw-FcrrjEMW",
        "colab_type": "text"
      },
      "source": [
        "# LSTM Final Model: Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FsT12SSbmzAl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.callbacks import ReduceLROnPlateau\n",
        "call_reduce = ReduceLROnPlateau(monitor='val_acc', factor=0.95, patience=3, verbose=2,\n",
        "                                mode='auto', min_delta=0.01, cooldown=0, min_lr=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bDysNyfIm7Em",
        "colab_type": "code",
        "outputId": "8b4e9e52-ff19-44a7-b2dc-e7a2bafd681f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        }
      },
      "source": [
        "model.fit_generator(train_generator(df_train), steps_per_epoch=batches_per_epoch, epochs=10,\n",
        "                    validation_data=val_generator(df_val), validation_steps=batches_per_epoch_val, callbacks =[call_reduce] )"
      ],
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "4571/4571 [==============================] - 61s 13ms/step - loss: 0.3352 - acc: 0.9113 - val_loss: 0.3816 - val_acc: 0.8979\n",
            "Epoch 2/10\n",
            "4571/4571 [==============================] - 60s 13ms/step - loss: 0.3059 - acc: 0.9150 - val_loss: 0.3866 - val_acc: 0.8906\n",
            "Epoch 3/10\n",
            "4571/4571 [==============================] - 61s 13ms/step - loss: 0.2946 - acc: 0.9184 - val_loss: 0.3774 - val_acc: 0.8960\n",
            "Epoch 4/10\n",
            "4571/4571 [==============================] - 59s 13ms/step - loss: 0.2865 - acc: 0.9189 - val_loss: 0.3817 - val_acc: 0.8969\n",
            "\n",
            "Epoch 00004: ReduceLROnPlateau reducing learning rate to 0.0009500000451225787.\n",
            "Epoch 5/10\n",
            "4571/4571 [==============================] - 60s 13ms/step - loss: 0.2805 - acc: 0.9202 - val_loss: 0.3752 - val_acc: 0.8979\n",
            "Epoch 6/10\n",
            "4571/4571 [==============================] - 60s 13ms/step - loss: 0.2748 - acc: 0.9210 - val_loss: 0.3730 - val_acc: 0.8960\n",
            "Epoch 7/10\n",
            "4571/4571 [==============================] - 59s 13ms/step - loss: 0.2720 - acc: 0.9215 - val_loss: 0.3716 - val_acc: 0.8989\n",
            "\n",
            "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.0009025000152178108.\n",
            "Epoch 8/10\n",
            "4571/4571 [==============================] - 58s 13ms/step - loss: 0.2631 - acc: 0.9239 - val_loss: 0.3827 - val_acc: 0.8921\n",
            "Epoch 9/10\n",
            "4571/4571 [==============================] - 60s 13ms/step - loss: 0.2600 - acc: 0.9240 - val_loss: 0.3802 - val_acc: 0.9003\n",
            "Epoch 10/10\n",
            "4571/4571 [==============================] - 60s 13ms/step - loss: 0.2549 - acc: 0.9255 - val_loss: 0.3920 - val_acc: 0.8969\n",
            "\n",
            "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.0008573750033974647.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f1f51d98d68>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6h_RWqTXjMZX",
        "colab_type": "text"
      },
      "source": [
        "# LSTM Final Model: Evaluation\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UfXOHliiNzDT",
        "colab_type": "code",
        "outputId": "b88421b2-936a-45cc-c472-23d0154a16fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "num_sequences_val = len(df_test['emb'].to_list())\n",
        "batch_size_val = 4\n",
        "batches_per_epoch_val = 343\n",
        "assert batch_size_val * batches_per_epoch_val == num_sequences_val\n",
        "num_features= 768\n",
        "model.evaluate_generator(val_generator(df_test), steps= batches_per_epoch_val)"
      ],
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.41612950315069047, 0.8731778425655977]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    }
  ]
}